From 9bacb1c76903afbbce4cbc333131f10517f51f55 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 13:19:17 +0300 Subject: [PATCH 01/18] [tshev] innocent performance improvements --- Makefile | 3 ++- setup.py | 16 +++++++++++++--- src/densematrix.h | 14 +++++++------- src/dictionary.cc | 2 ++ src/fasttext.cc | 31 +++++++++++++++---------------- src/loss.cc | 8 ++++---- src/loss.h | 16 ++++++++-------- src/quantmatrix.h | 2 +- 8 files changed, 52 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index ea5c78f..0207a08 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ # CXX = c++ -CXXFLAGS = -pthread -std=c++0x -march=native +CXXFLAGS = -Wall -pthread -std=c++14 -march=native -ffast-math -Wsuggest-final-methods -Walloc-zero -Wsuggest-override -Wodr -flto -ftree-loop-linear -floop-strip-mine -floop-block + OBJS = args.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o INCLUDES = -I. diff --git a/setup.py b/setup.py index d8061c3..1b2a961 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,10 @@ def __str__(self): map(lambda x: str(os.path.join(FASTTEXT_SRC, x)), fasttext_src_cc) ) +extra_compile_args = " -march=native -ffast-math -Wsuggest-final-methods" \ + " -Walloc-zero -Wsuggest-override -Wodr -flto -ftree-loop-linear" \ + " -floop-strip-mine -floop-block " + ext_modules = [ Extension( str('fasttext_pybind'), @@ -74,8 +78,8 @@ def __str__(self): FASTTEXT_SRC, ], language='c++', - extra_compile_args=["-O0 -fno-inline -fprofile-arcs -pthread -march=native" if coverage else - "-O3 -funroll-loops -pthread -march=native"], + extra_compile_args=[("-O0 -fno-inline -fprofile-arcs -pthread -march=native" if coverage else + "-O3 -funroll-loops -pthread -march=native") + extra_compile_args], ), ] @@ -100,6 +104,7 @@ def cpp_flag(compiler): """Return the -std=c++[0x/11/14] compiler flag. The c++14 is preferred over c++0x/11 (when it is available). """ + return '-std=c++14' standards = ['-std=c++14', '-std=c++11', '-std=c++0x'] for standard in standards: if has_flag(compiler, [standard]): @@ -134,6 +139,11 @@ def build_extensions(self): ct = self.compiler.compiler_type opts = self.c_opts.get(ct, []) extra_link_args = [] + self.c_opts['unix'] += [ + "-flto", "-march=native", "-ffast-math", "-Wsuggest-final-methods", + "-Walloc-zero", "-Wsuggest-override", "-Wodr", "-ftree-loop-linear", + "-floop-strip-mine", "-floop-block", + ] if coverage: coverage_option = '--coverage' @@ -190,7 +200,7 @@ def _get_readme(): 'Operating System :: Unix', 'Operating System :: MacOS', ], - install_requires=['pybind11>=2.2', "setuptools >= 0.7.0", "numpy"], + install_requires=[], cmdclass={'build_ext': BuildExt}, packages=[ str('fastText'), diff --git a/src/densematrix.h b/src/densematrix.h index 1296779..be2d024 100644 --- a/src/densematrix.h +++ b/src/densematrix.h @@ -32,7 +32,7 @@ class DenseMatrix : public Matrix { DenseMatrix(DenseMatrix&&) noexcept; DenseMatrix& operator=(const DenseMatrix&) = delete; DenseMatrix& operator=(DenseMatrix&&) = delete; - virtual ~DenseMatrix() noexcept override = default; + virtual ~DenseMatrix() noexcept final override = default; inline real* data() { return data_.data(); @@ -64,12 +64,12 @@ class DenseMatrix : public Matrix { real l2NormRow(int64_t i) const; void l2NormRow(Vector& norms) const; - real dotRow(const Vector&, int64_t) const override; - void addVectorToRow(const Vector&, int64_t, real) override; + real dotRow(const Vector&, int64_t) const override final; + void addVectorToRow(const Vector&, int64_t, real) override final; void addRowToVector(Vector& x, int32_t i) const override; - void addRowToVector(Vector& x, int32_t i, real a) const override; - void save(std::ostream&) const override; - void load(std::istream&) override; - void dump(std::ostream&) const override; + void addRowToVector(Vector& x, int32_t i, real a) const override final; + void save(std::ostream&) const override final; + void load(std::istream&) override final; + void dump(std::ostream&) const override final; }; } // namespace fasttext diff --git a/src/dictionary.cc b/src/dictionary.cc index cb396cd..16fd1db 100644 --- a/src/dictionary.cc +++ b/src/dictionary.cc @@ -489,6 +489,8 @@ void Dictionary::init() { void Dictionary::prune(std::vector& idx) { std::vector words, ngrams; + words.reserve(idx.size()); + ngrams.reserve(idx.size()); for (auto it = idx.cbegin(); it != idx.cend(); ++it) { if (*it < nwords_) { words.push_back(*it); diff --git a/src/fasttext.cc b/src/fasttext.cc index 2fb9247..7ff373c 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -463,9 +463,9 @@ bool FastText::predictLine( dict_->getLine(in, words, labels); Predictions linePredictions; predict(k, words, linePredictions, threshold); + predictions.reserve(linePredictions.size()); for (const auto& p : linePredictions) { - predictions.push_back( - std::make_pair(std::exp(p.first), dict_->getLabel(p.second))); + predictions.emplace_back(std::exp(p.first), dict_->getLabel(p.second)); } return true; @@ -511,12 +511,13 @@ std::vector> FastText::getNgramVectors( std::vector substrings; dict_->getSubwords(word, ngrams, substrings); assert(ngrams.size() <= substrings.size()); + result.reserve(ngrams.size()); for (int32_t i = 0; i < ngrams.size(); i++) { Vector vec(args_->dim); if (ngrams[i] >= 0) { vec.addRow(*input_, ngrams[i]); } - result.push_back(std::make_pair(substrings[i], std::move(vec))); + result.emplace_back(substrings[i], std::move(vec)); } return result; } @@ -585,7 +586,7 @@ std::vector> FastText::getNN( if (heap.size() == k && similarity < heap.front().first) { continue; } - heap.push_back(std::make_pair(similarity, word)); + heap.emplace_back(similarity, word); std::push_heap(heap.begin(), heap.end(), comparePairs); if (heap.size() > k) { std::pop_heap(heap.begin(), heap.end(), comparePairs); @@ -686,7 +687,6 @@ std::shared_ptr FastText::getInputMatrixFromFile( const std::string& filename) const { std::ifstream in(filename); std::vector words; - std::shared_ptr mat; // temp. matrix for pretrained vectors int64_t n, dim; if (!in.is_open()) { throw std::invalid_argument(filename + " cannot be opened for loading!"); @@ -697,23 +697,22 @@ std::shared_ptr FastText::getInputMatrixFromFile( "Dimension of pretrained vectors (" + std::to_string(dim) + ") does not match dimension (" + std::to_string(args_->dim) + ")!"); } - mat = std::make_shared(n, dim); + DenseMatrix mat(n, dim); // temp. matrix for pretrained vectors for (size_t i = 0; i < n; i++) { std::string word; in >> word; - words.push_back(word); + words.push_back(std::move(word)); dict_->add(word); for (size_t j = 0; j < dim; j++) { - in >> mat->at(i, j); + in >> mat.at(i, j); } } in.close(); dict_->threshold(1, 0); dict_->init(); - std::shared_ptr input = std::make_shared( - dict_->nwords() + args_->bucket, args_->dim); - input->uniform(1.0 / args_->dim); + DenseMatrix input(dict_->nwords() + args_->bucket, args_->dim); + input.uniform(1.0 / args_->dim); for (size_t i = 0; i < n; i++) { int32_t idx = dict_->getId(words[i]); @@ -721,10 +720,10 @@ std::shared_ptr FastText::getInputMatrixFromFile( continue; } for (size_t j = 0; j < dim; j++) { - input->at(idx, j) = mat->at(i, j); + input.at(idx, j) = mat.at(i, j); } } - return input; + return std::make_shared(std::move(input)); } void FastText::loadVectors(const std::string& filename) { @@ -781,6 +780,7 @@ void FastText::startThreads() { tokenCount_ = 0; loss_ = -1; std::vector threads; + threads.reserve(args_->thread); for (int32_t i = 0; i < args_->thread; i++) { threads.push_back(std::thread([=]() { trainThread(i); })); } @@ -812,9 +812,8 @@ bool FastText::isQuant() const { return quant_; } -bool comparePairs( - const std::pair& l, - const std::pair& r) { +bool comparePairs(const std::pair &l, + const std::pair &r) { return l.first > r.first; } diff --git a/src/loss.cc b/src/loss.cc index 285eb9f..98809d9 100644 --- a/src/loss.cc +++ b/src/loss.cc @@ -83,7 +83,7 @@ void Loss::findKBest( if (heap.size() == k && std_log(output[i]) < heap.front().first) { continue; } - heap.push_back(std::make_pair(std_log(output[i]), i)); + heap.emplace_back(std_log(output[i]), i); std::push_heap(heap.begin(), heap.end(), comparePairs); if (heap.size() > k) { std::pop_heap(heap.begin(), heap.end(), comparePairs); @@ -239,8 +239,8 @@ void HierarchicalSoftmaxLoss::buildTree(const std::vector& counts) { code.push_back(tree_[j].binary); j = tree_[j].parent; } - paths_.push_back(path); - codes_.push_back(code); + paths_.push_back(std::move(path)); + codes_.push_back(std::move(code)); } } @@ -284,7 +284,7 @@ void HierarchicalSoftmaxLoss::dfs( } if (tree_[node].left == -1 && tree_[node].right == -1) { - heap.push_back(std::make_pair(score, node)); + heap.emplace_back(score, node); std::push_heap(heap.begin(), heap.end(), comparePairs); if (heap.size() > k) { std::pop_heap(heap.begin(), heap.end(), comparePairs); diff --git a/src/loss.h b/src/loss.h index 3aea72f..7fe173c 100644 --- a/src/loss.h +++ b/src/loss.h @@ -73,7 +73,7 @@ class BinaryLogisticLoss : public Loss { class OneVsAllLoss : public BinaryLogisticLoss { public: explicit OneVsAllLoss(std::shared_ptr& wo); - ~OneVsAllLoss() noexcept override = default; + ~OneVsAllLoss() noexcept override final = default; real forward( const std::vector& targets, int32_t targetIndex, @@ -96,14 +96,14 @@ class NegativeSamplingLoss : public BinaryLogisticLoss { std::shared_ptr& wo, int neg, const std::vector& targetCounts); - ~NegativeSamplingLoss() noexcept override = default; + ~NegativeSamplingLoss() noexcept override final = default; real forward( const std::vector& targets, int32_t targetIndex, Model::State& state, real lr, - bool backprop) override; + bool backprop) override final; }; class HierarchicalSoftmaxLoss : public BinaryLogisticLoss { @@ -133,31 +133,31 @@ class HierarchicalSoftmaxLoss : public BinaryLogisticLoss { explicit HierarchicalSoftmaxLoss( std::shared_ptr& wo, const std::vector& counts); - ~HierarchicalSoftmaxLoss() noexcept override = default; + ~HierarchicalSoftmaxLoss() noexcept override final = default; real forward( const std::vector& targets, int32_t targetIndex, Model::State& state, real lr, - bool backprop) override; + bool backprop) override final; void predict( int32_t k, real threshold, Predictions& heap, - Model::State& state) const override; + Model::State& state) const override final; }; class SoftmaxLoss : public Loss { public: explicit SoftmaxLoss(std::shared_ptr& wo); - ~SoftmaxLoss() noexcept override = default; + ~SoftmaxLoss() noexcept override final = default; real forward( const std::vector& targets, int32_t targetIndex, Model::State& state, real lr, bool backprop) override; - void computeOutput(Model::State& state) const override; + void computeOutput(Model::State& state) const override final; }; } // namespace fasttext diff --git a/src/quantmatrix.h b/src/quantmatrix.h index 9e1b2f2..ffe8eba 100644 --- a/src/quantmatrix.h +++ b/src/quantmatrix.h @@ -43,7 +43,7 @@ class QuantMatrix : public Matrix { QuantMatrix(QuantMatrix&&) = delete; QuantMatrix& operator=(const QuantMatrix&) = delete; QuantMatrix& operator=(QuantMatrix&&) = delete; - virtual ~QuantMatrix() noexcept override = default; + virtual ~QuantMatrix() noexcept override final = default; void quantizeNorm(const Vector&); void quantize(DenseMatrix&& mat); From a5584fb96cb0ab3d0ad3ab5a72a4381d2a702826 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 13:21:48 +0300 Subject: [PATCH 02/18] [tshev] innocent performance improvements --- src/args.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/args.cc b/src/args.cc index cc1af1b..29de668 100644 --- a/src/args.cc +++ b/src/args.cc @@ -175,7 +175,7 @@ void Args::parseArgs(const std::vector& args) { printHelp(); exit(EXIT_FAILURE); } - } catch (std::out_of_range) { + } catch (std::out_of_range&) { std::cerr << args[ai] << " is missing an argument" << std::endl; printHelp(); exit(EXIT_FAILURE); From e92e93b0941b3a940ed10cc1d50298dc4ff2be47 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 15:07:26 +0300 Subject: [PATCH 03/18] [tshev] innocent performance improvements in pybind --- python/fastText/pybind/fasttext_pybind.cc | 92 ++++++++++++----------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/python/fastText/pybind/fasttext_pybind.cc b/python/fastText/pybind/fasttext_pybind.cc index 76c25b8..526ab1a 100644 --- a/python/fastText/pybind/fasttext_pybind.cc +++ b/python/fastText/pybind/fasttext_pybind.cc @@ -39,7 +39,7 @@ py::str castToPythonString(const std::string& s, const char* onUnicodeError) { std::pair, std::vector> getLineText( fasttext::FastText& m, - const std::string text, + const std::string& text, const char* onUnicodeError) { std::shared_ptr d = m.getDictionary(); std::stringstream ioss(text); @@ -60,7 +60,7 @@ std::pair, std::vector> getLineText( if (token == fasttext::Dictionary::EOS) break; } - return std::pair, std::vector>(words, labels); + return {std::move(words), std::move(labels)}; } PYBIND11_MODULE(fasttext_pybind, m) { @@ -159,13 +159,13 @@ PYBIND11_MODULE(fasttext_pybind, m) { }) .def( "loadModel", - [](fasttext::FastText& m, std::string s) { m.loadModel(s); }) + [](fasttext::FastText& m, const std::string& s) { m.loadModel(s); }) .def( "saveModel", - [](fasttext::FastText& m, std::string s) { m.saveModel(s); }) + [](fasttext::FastText& m, const std::string& s) { m.saveModel(s); }) .def( "test", - [](fasttext::FastText& m, const std::string filename, int32_t k) { + [](fasttext::FastText& m, const std::string& filename, int32_t k) { std::ifstream ifs(filename); if (!ifs.is_open()) { throw std::invalid_argument("Test file cannot be opened!"); @@ -180,13 +180,13 @@ PYBIND11_MODULE(fasttext_pybind, m) { "getSentenceVector", [](fasttext::FastText& m, fasttext::Vector& v, - const std::string text) { + const std::string& text) { std::stringstream ioss(text); m.getSentenceVector(ioss, v); }) .def( "tokenize", - [](fasttext::FastText& m, const std::string text) { + [](fasttext::FastText& m, const std::string& text) { std::vector text_split; std::shared_ptr d = m.getDictionary(); std::stringstream ioss(text); @@ -202,53 +202,57 @@ PYBIND11_MODULE(fasttext_pybind, m) { .def( "multilineGetLine", [](fasttext::FastText& m, - const std::vector lines, + const std::vector& lines, const char* onUnicodeError) { std::shared_ptr d = m.getDictionary(); + std::vector> all_words; + all_words.reserve(lines.size()); std::vector> all_labels; + all_labels.reserve(lines.size()); + for (const auto& text : lines) { auto pair = getLineText(m, text, onUnicodeError); - all_words.push_back(pair.first); - all_labels.push_back(pair.second); + all_words.push_back(std::move(pair.first)); + all_labels.push_back(std::move(pair.second)); } return std::pair< std::vector>, - std::vector>>(all_words, all_labels); + std::vector>>(std::move(all_words), std::move(all_labels)); }) .def( "getVocab", [](fasttext::FastText& m, const char* onUnicodeError) { py::str s; - std::vector vocab_list; - std::vector vocab_freq; std::shared_ptr d = m.getDictionary(); - vocab_freq = d->getCounts(fasttext::entry_type::word); + std::vector vocab_freq = d->getCounts(fasttext::entry_type::word); + std::vector vocab_list; + vocab_list.reserve(vocab_freq.size()); for (int32_t i = 0; i < vocab_freq.size(); i++) { - vocab_list.push_back( - castToPythonString(d->getWord(i), onUnicodeError)); + vocab_list.push_back(castToPythonString(d->getWord(i), onUnicodeError)); } return std::pair, std::vector>( - vocab_list, vocab_freq); + std::move(vocab_list), std::move(vocab_freq)); }) .def( "getLabels", [](fasttext::FastText& m, const char* onUnicodeError) { - std::vector labels_list; - std::vector labels_freq; std::shared_ptr d = m.getDictionary(); - labels_freq = d->getCounts(fasttext::entry_type::label); + std::vector labels_freq = d->getCounts(fasttext::entry_type::label); + std::vector labels_list; + labels_list.reserve(labels_freq.size()); + for (int32_t i = 0; i < labels_freq.size(); i++) { labels_list.push_back( castToPythonString(d->getLabel(i), onUnicodeError)); } return std::pair, std::vector>( - labels_list, labels_freq); + std::move(labels_list), std::move(labels_freq)); }) .def( "quantize", [](fasttext::FastText& m, - const std::string input, + const std::string& input, bool qout, int32_t cutoff, bool retrain, @@ -276,7 +280,7 @@ PYBIND11_MODULE(fasttext_pybind, m) { // NOTE: text needs to end in a newline // to exactly mimic the behavior of the cli [](fasttext::FastText& m, - const std::string text, + const std::string& text, int32_t k, fasttext::real threshold, const char* onUnicodeError) { @@ -284,13 +288,14 @@ PYBIND11_MODULE(fasttext_pybind, m) { std::vector> predictions; m.predictLine(ioss, predictions, k, threshold); - std::vector> - transformedPredictions; + std::vector> transformedPredictions; + transformedPredictions.reserve(predictions.size()); for (const auto& prediction : predictions) { - transformedPredictions.push_back(std::make_pair( + transformedPredictions.emplace_back( prediction.first, - castToPythonString(prediction.second, onUnicodeError))); + castToPythonString(prediction.second, onUnicodeError) + ); } return transformedPredictions; @@ -306,26 +311,28 @@ PYBIND11_MODULE(fasttext_pybind, m) { const char* onUnicodeError) { std::vector>> allPredictions; + allPredictions.reserve(lines.size()); std::vector> predictions; for (const std::string& text : lines) { - std::stringstream ioss(text); + std::stringstream ioss(text); /// stringstream is slow m.predictLine(ioss, predictions, k, threshold); - std::vector> - transformedPredictions; + std::vector> transformedPredictions; + transformedPredictions.reserve(predictions.size()); for (const auto& prediction : predictions) { - transformedPredictions.push_back(std::make_pair( + transformedPredictions.emplace_back( prediction.first, - castToPythonString(prediction.second, onUnicodeError))); + castToPythonString(prediction.second, onUnicodeError) + ); } - allPredictions.push_back(transformedPredictions); + allPredictions.push_back(std::move(transformedPredictions)); } return allPredictions; }) .def( "testLabel", [](fasttext::FastText& m, - const std::string filename, + const std::string& filename, int32_t k, fasttext::real threshold) { std::ifstream ifs(filename); @@ -335,7 +342,7 @@ PYBIND11_MODULE(fasttext_pybind, m) { fasttext::Meter meter; m.test(ifs, k, threshold, meter); std::shared_ptr d = m.getDictionary(); - std::unordered_map returnedValue; + std::unordered_map returnedValue(d->nlabels()); for (int32_t i = 0; i < d->nlabels(); i++) { returnedValue[d->getLabel(i)] = py::dict( "precision"_a = meter.precision(i), @@ -347,12 +354,12 @@ PYBIND11_MODULE(fasttext_pybind, m) { }) .def( "getWordId", - [](fasttext::FastText& m, const std::string word) { + [](fasttext::FastText& m, const std::string& word) { return m.getWordId(word); }) .def( "getSubwordId", - [](fasttext::FastText& m, const std::string word) { + [](fasttext::FastText& m, const std::string& word) { return m.getSubwordId(word); }) .def( @@ -364,25 +371,26 @@ PYBIND11_MODULE(fasttext_pybind, m) { "getWordVector", [](fasttext::FastText& m, fasttext::Vector& vec, - const std::string word) { m.getWordVector(vec, word); }) + const std::string& word) { m.getWordVector(vec, word); }) .def( "getSubwords", [](fasttext::FastText& m, - const std::string word, + const std::string& word, const char* onUnicodeError) { std::vector subwords; std::vector ngrams; std::shared_ptr d = m.getDictionary(); d->getSubwords(word, ngrams, subwords); + std::vector transformedSubwords; + transformedSubwords.reserve(subwords.size()); for (const auto& subword : subwords) { - transformedSubwords.push_back( - castToPythonString(subword, onUnicodeError)); + transformedSubwords.push_back(castToPythonString(subword, onUnicodeError)); } return std::pair, std::vector>( - transformedSubwords, ngrams); + std::move(transformedSubwords), std::move(ngrams)); }) .def("isQuant", [](fasttext::FastText& m) { return m.isQuant(); }); } From 721b6cd3a0aee414483a574db0774696a30a63fa Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 15:41:45 +0300 Subject: [PATCH 04/18] [tshev] O3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b2a961..0972a2d 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ def build_extensions(self): self.c_opts['unix'] += [ "-flto", "-march=native", "-ffast-math", "-Wsuggest-final-methods", "-Walloc-zero", "-Wsuggest-override", "-Wodr", "-ftree-loop-linear", - "-floop-strip-mine", "-floop-block", + "-floop-strip-mine", "-floop-block", "-O3", ] if coverage: From 1c24f5eec59688255f7d667172e6ab3a3a695e75 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 17:52:07 +0300 Subject: [PATCH 05/18] [tshev] add a notice for performance penalty --- src/fasttext.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fasttext.cc b/src/fasttext.cc index 7ff373c..34adf73 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -472,6 +472,7 @@ bool FastText::predictLine( } void FastText::getSentenceVector(std::istream& in, fasttext::Vector& svec) { + // std::istream is slow svec.zero(); if (args_->model == model_name::sup) { std::vector line, labels; From de34e16491b3a47e55a1206eb4ec613ff4b8b702 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 21:03:49 +0300 Subject: [PATCH 06/18] [tshev] improve code style --- src/dictionary.cc | 32 +++++++++++++++++++------------- src/fasttext.cc | 6 +++--- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/dictionary.cc b/src/dictionary.cc index 16fd1db..ea81700 100644 --- a/src/dictionary.cc +++ b/src/dictionary.cc @@ -161,23 +161,27 @@ std::string Dictionary::getWord(int32_t id) const { // using signed char, we fixed the hash function to make models // compatible whatever compiler is used. uint32_t Dictionary::hash(const std::string& str) const { - uint32_t h = 2166136261; - for (size_t i = 0; i < str.size(); i++) { - h = h ^ uint32_t(int8_t(str[i])); - h = h * 16777619; + uint32_t h = 2166136261u; + for (auto x : str) { + h = h ^ uint32_t(int8_t(x)); // TODO: oops + h = h * 16777619u; } return h; } + void Dictionary::computeSubwords( const std::string& word, std::vector& ngrams, std::vector* substrings) const { + + ngrams.reserve(3ul * word.size()); for (size_t i = 0; i < word.size(); i++) { std::string ngram; if ((word[i] & 0xC0) == 0x80) { continue; } + for (size_t j = i, n = 1; j < word.size() && n <= args_->maxn; n++) { ngram.push_back(word[j++]); while (j < word.size() && (word[j] & 0xC0) == 0x80) { @@ -301,11 +305,13 @@ void Dictionary::initTableDiscard() { std::vector Dictionary::getCounts(entry_type type) const { std::vector counts; - for (auto& w : words_) { + // counts.reserve(words_.size()); + for (const auto& w : words_) { if (w.type == type) { counts.push_back(w.count); } } + // counts.shrink_to_fit(); return counts; } @@ -491,11 +497,11 @@ void Dictionary::prune(std::vector& idx) { std::vector words, ngrams; words.reserve(idx.size()); ngrams.reserve(idx.size()); - for (auto it = idx.cbegin(); it != idx.cend(); ++it) { - if (*it < nwords_) { - words.push_back(*it); + for (int32_t x : idx) { + if (x < nwords_) { + words.push_back(x); } else { - ngrams.push_back(*it); + ngrams.push_back(x); } } std::sort(words.begin(), words.end()); @@ -530,12 +536,12 @@ void Dictionary::prune(std::vector& idx) { void Dictionary::dump(std::ostream& out) const { out << words_.size() << std::endl; - for (auto it : words_) { - std::string entryType = "word"; + for (const auto& it : words_) { if (it.type == entry_type::label) { - entryType = "label"; + out << it.word << " " << it.count << " " << "label" << std::endl; + } else { + out << it.word << " " << it.count << " " << "word" << std::endl; } - out << it.word << " " << it.count << " " << entryType << std::endl; } } diff --git a/src/fasttext.cc b/src/fasttext.cc index 34adf73..222625a 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -89,9 +89,10 @@ int32_t FastText::getSubwordId(const std::string& subword) const { void FastText::getWordVector(Vector& vec, const std::string& word) const { const std::vector& ngrams = dict_->getSubwords(word); vec.zero(); - for (int i = 0; i < ngrams.size(); i++) { - addInputVector(vec, ngrams[i]); + for (int32_t ngram : ngrams) { + addInputVector(vec, ngram); } + if (ngrams.size() > 0) { vec.mul(1.0 / ngrams.size()); } @@ -121,7 +122,6 @@ void FastText::saveVectors(const std::string& filename) { getWordVector(vec, word); ofs << word << " " << vec << std::endl; } - ofs.close(); } void FastText::saveVectors() { From e32069f8c19df5148a0d0e26fc20add01580f3a6 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 22:42:38 +0300 Subject: [PATCH 07/18] [tshev] fix FastText::getNN --- src/fasttext.cc | 31 +++++++++++++++++++++---------- src/fasttext.h | 7 +++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/fasttext.cc b/src/fasttext.cc index 222625a..ad15147 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -572,30 +572,41 @@ std::vector> FastText::getNN( int32_t k, const std::set& banSet) { std::vector> heap; + heap.reserve(size_t(k + 1)); real queryNorm = query.norm(); if (std::abs(queryNorm) < 1e-8) { queryNorm = 1; } - for (int32_t i = 0; i < dict_->nwords(); i++) { + int32_t i; + for (i = 0; i < dict_->nwords() && heap.size() < k; ++i) { std::string word = dict_->getWord(i); if (banSet.find(word) == banSet.end()) { real dp = wordVectors.dotRow(query, i); real similarity = dp / queryNorm; + heap.emplace_back(similarity, std::move(word)); + } + } + greater_first cmp; + std::make_heap(std::begin(heap), std::end(heap), cmp); - if (heap.size() == k && similarity < heap.front().first) { - continue; - } - heap.emplace_back(similarity, word); - std::push_heap(heap.begin(), heap.end(), comparePairs); - if (heap.size() > k) { - std::pop_heap(heap.begin(), heap.end(), comparePairs); - heap.pop_back(); + for (; i < dict_->nwords(); ++i) { + std::string word = dict_->getWord(i); + if (banSet.find(word) == banSet.end()) { + real dp = wordVectors.dotRow(query, i); + real similarity = dp / queryNorm; + + if (similarity >= heap.front().first) { + heap.emplace_back(similarity, std::move(word)); + std::push_heap(heap.begin(), heap.end(), cmp); + std::pop_heap(heap.begin(), heap.end(), cmp); + heap.pop_back(); } } } - std::sort_heap(heap.begin(), heap.end(), comparePairs); + + std::sort(heap.begin(), heap.end(), cmp); // faster than std::sort_heap return heap; } diff --git a/src/fasttext.h b/src/fasttext.h index b63e4cd..a4e8387 100644 --- a/src/fasttext.h +++ b/src/fasttext.h @@ -188,4 +188,11 @@ class FastText { const std::set& banSet, std::vector>& results); }; + +template +struct greater_first { + bool operator()(const T& x, const T& y) { + return y.first < x.first; + } +}; } // namespace fasttext From 7c1445deb58f762ef3e096cf7ad4afba0f728cbc Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 1 May 2019 23:04:10 +0300 Subject: [PATCH 08/18] [tshev] replace for loop with a while loop --- src/fasttext.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fasttext.cc b/src/fasttext.cc index ad15147..a1704b3 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -579,19 +579,21 @@ std::vector> FastText::getNN( queryNorm = 1; } - int32_t i; - for (i = 0; i < dict_->nwords() && heap.size() < k; ++i) { + int32_t i = 0; + while (i < dict_->nwords() && heap.size() < k) { std::string word = dict_->getWord(i); if (banSet.find(word) == banSet.end()) { real dp = wordVectors.dotRow(query, i); real similarity = dp / queryNorm; heap.emplace_back(similarity, std::move(word)); } + ++i; } + greater_first cmp; std::make_heap(std::begin(heap), std::end(heap), cmp); - for (; i < dict_->nwords(); ++i) { + while (i < dict_->nwords()) { std::string word = dict_->getWord(i); if (banSet.find(word) == banSet.end()) { real dp = wordVectors.dotRow(query, i); @@ -604,6 +606,7 @@ std::vector> FastText::getNN( heap.pop_back(); } } + ++i; } std::sort(heap.begin(), heap.end(), cmp); // faster than std::sort_heap From 124649ed27b0be4d14f56a9dc487b96401355a9b Mon Sep 17 00:00:00 2001 From: tshev Date: Thu, 2 May 2019 11:35:50 +0300 Subject: [PATCH 09/18] [tshev] add back requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0972a2d..5e277a6 100644 --- a/setup.py +++ b/setup.py @@ -200,7 +200,7 @@ def _get_readme(): 'Operating System :: Unix', 'Operating System :: MacOS', ], - install_requires=[], + install_requires=['pybind11>=2.2', "setuptools >= 0.7.0", "numpy"], cmdclass={'build_ext': BuildExt}, packages=[ str('fastText'), From 1f0ff4a63552248508e0f7eee162b3e9990eac34 Mon Sep 17 00:00:00 2001 From: tshev Date: Thu, 2 May 2019 11:40:10 +0300 Subject: [PATCH 10/18] [tshev] update gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 690aeaf..965d9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ data fasttext result +build/ +python/fasttext.egg-info/ +python/fasttext_pybind.cpython-37m-x86_64-linux-gnu.so + From 3f3ee8da15ca2ca25101fd1043a4525b36a1ca82 Mon Sep 17 00:00:00 2001 From: Taras Shevchenko Date: Wed, 8 May 2019 14:50:38 +0300 Subject: [PATCH 11/18] [tshev/probs-for-all-the-labels] get probablities for all the labels (#2) * [tshev/probs-for-all-the-labels] get probablities for all the labels * [tshev/probs-for-all-the-labels] use better names * [tshev/probs-for-all-the-labels] use better names * [tshev/probs-for-all-the-labels] use better names * [tshev/probs-for-all-the-labels] use better names * [tshev/probs-for-all-the-labels] remove useless commit * [tshev/probs-for-all-the-labels] sort by label name --- python/fastText/FastText.py | 19 +++++++++- python/fastText/pybind/fasttext_pybind.cc | 46 +++++++++++++++++++++++ src/fasttext.cc | 33 ++++++++++++++++ src/fasttext.h | 4 ++ src/loss.cc | 33 ++++++++++++++++ src/loss.h | 6 +++ src/model.cc | 5 +++ src/model.h | 3 ++ 8 files changed, 148 insertions(+), 1 deletion(-) diff --git a/python/fastText/FastText.py b/python/fastText/FastText.py index afa9e85..9d530b6 100644 --- a/python/fastText/FastText.py +++ b/python/fastText/FastText.py @@ -139,9 +139,26 @@ def check(entry): text = check(text) predictions = self.f.predict(text, k, threshold, on_unicode_error) probs, labels = zip(*predictions) - return labels, np.array(probs, copy=False) + def predict_all(self, text, on_unicode_error='strict'): + def check(entry): + if entry.find('\n') != -1: + raise ValueError( + "predict processes one line at a time (remove \'\\n\')" + ) + entry += "\n" + return entry + + if type(text) is list: + text = [check(entry) for entry in text] + predictions = self.f.multilinePredictAll(text) + return np.array(predictions, dtype=float) + else: + text = check(text) + probs = self.f.predictAll(text) + return np.array(probs, copy=False) + def get_input_matrix(self): """ Get a copy of the full input matrix of a Model. This only diff --git a/python/fastText/pybind/fasttext_pybind.cc b/python/fastText/pybind/fasttext_pybind.cc index 526ab1a..00d062a 100644 --- a/python/fastText/pybind/fasttext_pybind.cc +++ b/python/fastText/pybind/fasttext_pybind.cc @@ -301,6 +301,27 @@ PYBIND11_MODULE(fasttext_pybind, m) { return transformedPredictions; }) .def( + "predictAll", + // NOTE: text needs to end in a newline + // to exactly mimic the behavior of the cli + [](fasttext::FastText& m, const std::string& text) { + std::stringstream ioss(text); + std::vector> predictions; + + m.predictLine(ioss, predictions); + std::sort(std::begin(predictions), std::end(predictions), [](const auto& x, const auto &y) { + return x.second < y.second; + }); + + std::vector transformedPredictions; + transformedPredictions.reserve(predictions.size()); + + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); + return transformedPredictions; + }) + .def( "multilinePredict", // NOTE: text needs to end in a newline // to exactly mimic the behavior of the cli @@ -329,6 +350,31 @@ PYBIND11_MODULE(fasttext_pybind, m) { } return allPredictions; }) + .def( + "multilinePredictAll", + // NOTE: text needs to end in a newline + // to exactly mimic the behavior of the cli + [](fasttext::FastText& m, const std::vector& lines) { + std::vector> allPredictions; + + allPredictions.reserve(lines.size()); + std::vector> predictions; + for (const std::string& text : lines) { + std::stringstream ioss(text); /// stringstream is slow + m.predictLine(ioss, predictions); + std::sort(std::begin(predictions), std::end(predictions), [](const auto& x, const auto &y) { + return x.second < y.second; + }); + std::vector transformedPredictions; + transformedPredictions.reserve(predictions.size()); + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); + allPredictions.push_back(std::move(transformedPredictions)); + } + return allPredictions; + }) + .def( "testLabel", [](fasttext::FastText& m, diff --git a/src/fasttext.cc b/src/fasttext.cc index a1704b3..fab8cf0 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -449,6 +449,20 @@ void FastText::predict( model_->predict(words, k, threshold, predictions, state); } +void FastText::predict(const std::vector& words, Predictions& predictions) const { + if (words.empty()) { + return; + } + Model::State state(args_->dim, dict_->nlabels(), 0); + predictions.reserve(dict_->nlabels()); + if (args_->model != model_name::sup) { + throw std::invalid_argument("Model needs to be supervised for prediction!"); + } + model_->predict(words, predictions, state); +} + + + bool FastText::predictLine( std::istream& in, std::vector>& predictions, @@ -471,6 +485,25 @@ bool FastText::predictLine( return true; } +bool FastText::predictLine(std::istream& in, std::vector>& predictions) const { + predictions.clear(); + if (in.peek() == EOF) { + return false; + } + + std::vector words, labels; + dict_->getLine(in, words, labels); + Predictions linePredictions; + predict(words, linePredictions); + predictions.reserve(linePredictions.size()); + for (const auto& p : linePredictions) { + predictions.emplace_back(std::exp(p.first), dict_->getLabel(p.second)); + } + + return true; +} + + void FastText::getSentenceVector(std::istream& in, fasttext::Vector& svec) { // std::istream is slow svec.zero(); diff --git a/src/fasttext.h b/src/fasttext.h index a4e8387..010ef45 100644 --- a/src/fasttext.h +++ b/src/fasttext.h @@ -122,12 +122,16 @@ class FastText { Predictions& predictions, real threshold = 0.0) const; + void predict(const std::vector& words, Predictions& predictions) const; + bool predictLine( std::istream& in, std::vector>& predictions, int32_t k, real threshold) const; + bool predictLine(std::istream& in, std::vector>& predictions) const; + std::vector> getNgramVectors( const std::string& word) const; diff --git a/src/loss.cc b/src/loss.cc index 98809d9..9610679 100644 --- a/src/loss.cc +++ b/src/loss.cc @@ -71,6 +71,15 @@ void Loss::predict( std::sort_heap(heap.begin(), heap.end(), comparePairs); } +void Loss::predict(Predictions& predictions, Model::State& state) const { + computeOutput(state); + const Vector& output = state.output; + predictions.reserve(output.size()); + for (int32_t i = 0; i < output.size(); i++) { + predictions.emplace_back(std_log(output[i]), i); + } +} + void Loss::findKBest( int32_t k, real threshold, @@ -269,6 +278,12 @@ void HierarchicalSoftmaxLoss::predict( std::sort_heap(heap.begin(), heap.end(), comparePairs); } +void HierarchicalSoftmaxLoss::predict(Predictions& predictions, Model::State& state) const { + dfs(2 * osz_ - 2, 0.0, predictions, state.hidden); +} + + + void HierarchicalSoftmaxLoss::dfs( int32_t k, real threshold, @@ -300,6 +315,24 @@ void HierarchicalSoftmaxLoss::dfs( dfs(k, threshold, tree_[node].right, score + std_log(f), heap, hidden); } +void HierarchicalSoftmaxLoss::dfs( + int32_t node, + real score, + Predictions& predictions, + const Vector& hidden) const { + + if (tree_[node].left == -1 && tree_[node].right == -1) { + predictions.emplace_back(score, node); + return; + } + + real f = wo_->dotRow(hidden, node - osz_); + f = 1. / (1 + std::exp(-f)); + + dfs(tree_[node].left, score + std_log(1.0 - f), predictions, hidden); + dfs(tree_[node].right, score + std_log(f), predictions, hidden); +} + SoftmaxLoss::SoftmaxLoss(std::shared_ptr& wo) : Loss(wo) {} void SoftmaxLoss::computeOutput(Model::State& state) const { diff --git a/src/loss.h b/src/loss.h index 7fe173c..d8e0d0e 100644 --- a/src/loss.h +++ b/src/loss.h @@ -53,6 +53,8 @@ class Loss { real /*threshold*/, Predictions& /*heap*/, Model::State& /*state*/) const; + + virtual void predict(Predictions& predictions, Model::State& state) const; }; class BinaryLogisticLoss : public Loss { @@ -129,6 +131,8 @@ class HierarchicalSoftmaxLoss : public BinaryLogisticLoss { Predictions& heap, const Vector& hidden) const; + void dfs(int32_t node, real score, Predictions& predictions, const Vector& hidden) const; + public: explicit HierarchicalSoftmaxLoss( std::shared_ptr& wo, @@ -145,6 +149,8 @@ class HierarchicalSoftmaxLoss : public BinaryLogisticLoss { real threshold, Predictions& heap, Model::State& state) const override final; + + void predict(Predictions& predictions, Model::State& state) const override final; }; class SoftmaxLoss : public Loss { diff --git a/src/model.cc b/src/model.cc index b4fcaed..a38c6f4 100644 --- a/src/model.cc +++ b/src/model.cc @@ -67,6 +67,11 @@ void Model::predict( loss_->predict(k, threshold, heap, state); } +void Model::predict(const std::vector& input, Predictions& predictions, State& state) const { + computeHidden(input, state); + loss_->predict(predictions, state); +} + void Model::update( const std::vector& input, const std::vector& targets, diff --git a/src/model.h b/src/model.h index 65987d6..35089df 100644 --- a/src/model.h +++ b/src/model.h @@ -62,6 +62,9 @@ class Model { real threshold, Predictions& heap, State& state) const; + + void predict(const std::vector& input, Predictions& predictions, State& state) const; + void update( const std::vector& input, const std::vector& targets, From f4fefa98b58127d9a72b217563a1680a3128feb2 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 8 May 2019 16:12:43 +0300 Subject: [PATCH 12/18] [faster-fastText] Remove warning for allocation of 0 bytes --- Makefile | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0207a08..d37a50a 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ # CXX = c++ -CXXFLAGS = -Wall -pthread -std=c++14 -march=native -ffast-math -Wsuggest-final-methods -Walloc-zero -Wsuggest-override -Wodr -flto -ftree-loop-linear -floop-strip-mine -floop-block +CXXFLAGS = -Wall -pthread -std=c++14 -march=native -ffast-math -Wsuggest-final-methods -Wsuggest-override -Wodr -flto -ftree-loop-linear -floop-strip-mine -floop-block OBJS = args.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o INCLUDES = -I. diff --git a/setup.py b/setup.py index 5e277a6..a8967eb 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def __str__(self): ) extra_compile_args = " -march=native -ffast-math -Wsuggest-final-methods" \ - " -Walloc-zero -Wsuggest-override -Wodr -flto -ftree-loop-linear" \ + " -Wsuggest-override -Wodr -flto -ftree-loop-linear" \ " -floop-strip-mine -floop-block " ext_modules = [ @@ -141,7 +141,7 @@ def build_extensions(self): extra_link_args = [] self.c_opts['unix'] += [ "-flto", "-march=native", "-ffast-math", "-Wsuggest-final-methods", - "-Walloc-zero", "-Wsuggest-override", "-Wodr", "-ftree-loop-linear", + "-Wsuggest-override", "-Wodr", "-ftree-loop-linear", "-floop-strip-mine", "-floop-block", "-O3", ] From 89f41da5e01d750bab851fbf624995e79de081ef Mon Sep 17 00:00:00 2001 From: tshev Date: Tue, 14 May 2019 23:54:35 +0300 Subject: [PATCH 13/18] [faster-fastText] longer delay between metadata updates --- src/fasttext.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fasttext.cc b/src/fasttext.cc index fab8cf0..43f53be 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -835,7 +835,7 @@ void FastText::startThreads() { const int64_t ntokens = dict_->ntokens(); // Same condition as trainThread while (tokenCount_ < args_->epoch * ntokens) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(10000)); if (loss_ >= 0 && args_->verbose > 1) { real progress = real(tokenCount_) / (args_->epoch * ntokens); std::cerr << "\r"; From 56bcc90b9bf0b5c80f144635e334fe3286c155c6 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 15 May 2019 22:14:41 +0300 Subject: [PATCH 14/18] [faster-fastText] Normalize vectors --- python/fastText/pybind/fasttext_pybind.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/python/fastText/pybind/fasttext_pybind.cc b/python/fastText/pybind/fasttext_pybind.cc index 00d062a..167d927 100644 --- a/python/fastText/pybind/fasttext_pybind.cc +++ b/python/fastText/pybind/fasttext_pybind.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -313,12 +314,22 @@ PYBIND11_MODULE(fasttext_pybind, m) { return x.second < y.second; }); + fasttext::real sum = std::accumulate(std::begin(predictions), std::end(predictions), fasttext::real{0.0}, [](fasttext::real r, const auto& x) { + return r + x.first; + }); + std::vector transformedPredictions; transformedPredictions.reserve(predictions.size()); - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { - return x.first; - }); + if (sum == fasttext::real(0.0)) { + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); + } else { + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [sum](const auto& x) { + return x.first / sum; + }); + } return transformedPredictions; }) .def( From b9614719defad6e08167acbb114dd3606eef0249 Mon Sep 17 00:00:00 2001 From: tshev Date: Wed, 15 May 2019 23:41:38 +0300 Subject: [PATCH 15/18] [faster-fastText] Normalize vectors --- python/fastText/pybind/fasttext_pybind.cc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/python/fastText/pybind/fasttext_pybind.cc b/python/fastText/pybind/fasttext_pybind.cc index 167d927..464797f 100644 --- a/python/fastText/pybind/fasttext_pybind.cc +++ b/python/fastText/pybind/fasttext_pybind.cc @@ -373,19 +373,31 @@ PYBIND11_MODULE(fasttext_pybind, m) { for (const std::string& text : lines) { std::stringstream ioss(text); /// stringstream is slow m.predictLine(ioss, predictions); + std::sort(std::begin(predictions), std::end(predictions), [](const auto& x, const auto &y) { return x.second < y.second; }); + + fasttext::real sum = std::accumulate(std::begin(predictions), std::end(predictions), fasttext::real{0.0}, [](fasttext::real r, const auto& x) { + return r + x.first; + }); + std::vector transformedPredictions; transformedPredictions.reserve(predictions.size()); - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { - return x.first; - }); + if (sum == fasttext::real(0.0)) { + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); + } else { + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [sum](const auto& x) { + return x.first / sum; + }); + } + allPredictions.push_back(std::move(transformedPredictions)); } return allPredictions; }) - .def( "testLabel", [](fasttext::FastText& m, From 4828ae22795cc5b030c957abc7d3dde6a2f3d6c4 Mon Sep 17 00:00:00 2001 From: tshev Date: Mon, 24 Jun 2019 15:37:56 +0300 Subject: [PATCH 16/18] fix an issue: https://github.com/tshev/faster-FastText/issues/5 --- src/fasttext.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fasttext.cc b/src/fasttext.cc index 43f53be..454dab2 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -749,7 +749,7 @@ std::shared_ptr FastText::getInputMatrixFromFile( for (size_t i = 0; i < n; i++) { std::string word; in >> word; - words.push_back(std::move(word)); + words.push_back(word); dict_->add(word); for (size_t j = 0; j < dim; j++) { in >> mat.at(i, j); From ce6f212ff8d9ec859d937fd55f3b5000e2728523 Mon Sep 17 00:00:00 2001 From: tshev Date: Tue, 2 Feb 2021 10:18:19 +0200 Subject: [PATCH 17/18] Remove the debug mode --- Makefile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d37a50a..12c4f5f 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ CXXFLAGS = -Wall -pthread -std=c++14 -march=native -ffast-math -Wsuggest-final-m OBJS = args.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o INCLUDES = -I. -opt: CXXFLAGS += -O3 -funroll-loops +opt: CXXFLAGS += -O3 -funroll-loops -DNDEBUG opt: fasttext coverage: CXXFLAGS += -O0 -fno-inline -fprofile-arcs --coverage diff --git a/setup.py b/setup.py index a8967eb..215dbe6 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ def build_extensions(self): self.c_opts['unix'] += [ "-flto", "-march=native", "-ffast-math", "-Wsuggest-final-methods", "-Wsuggest-override", "-Wodr", "-ftree-loop-linear", - "-floop-strip-mine", "-floop-block", "-O3", + "-floop-strip-mine", "-floop-block", "-O3", "-DNDEBUG", ] if coverage: From 4df58a3d0eabc6c0a1ad3a7311709e3b002a3712 Mon Sep 17 00:00:00 2001 From: tshev Date: Fri, 26 Sep 2025 00:30:31 +0300 Subject: [PATCH 18/18] Don't normalize predicted probabilities --- python/fastText/pybind/fasttext_pybind.cc | 33 +++++------------------ 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/python/fastText/pybind/fasttext_pybind.cc b/python/fastText/pybind/fasttext_pybind.cc index 464797f..dba074f 100644 --- a/python/fastText/pybind/fasttext_pybind.cc +++ b/python/fastText/pybind/fasttext_pybind.cc @@ -314,22 +314,12 @@ PYBIND11_MODULE(fasttext_pybind, m) { return x.second < y.second; }); - fasttext::real sum = std::accumulate(std::begin(predictions), std::end(predictions), fasttext::real{0.0}, [](fasttext::real r, const auto& x) { - return r + x.first; - }); - std::vector transformedPredictions; transformedPredictions.reserve(predictions.size()); - if (sum == fasttext::real(0.0)) { - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { - return x.first; - }); - } else { - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [sum](const auto& x) { - return x.first / sum; - }); - } + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); return transformedPredictions; }) .def( @@ -378,21 +368,12 @@ PYBIND11_MODULE(fasttext_pybind, m) { return x.second < y.second; }); - fasttext::real sum = std::accumulate(std::begin(predictions), std::end(predictions), fasttext::real{0.0}, [](fasttext::real r, const auto& x) { - return r + x.first; - }); - std::vector transformedPredictions; transformedPredictions.reserve(predictions.size()); - if (sum == fasttext::real(0.0)) { - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { - return x.first; - }); - } else { - std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [sum](const auto& x) { - return x.first / sum; - }); - } + + std::transform(std::begin(predictions), std::end(predictions), std::back_inserter(transformedPredictions), [](const auto& x) { + return x.first; + }); allPredictions.push_back(std::move(transformedPredictions)); }