Last active
February 19, 2021 17:53
-
-
Save jerinphilip/64679f42fa6929a3d2c860b16bb6f55e to your computer and use it in GitHub Desktop.
Revisions
-
jerinphilip revised this gist
Feb 19, 2021 . 1 changed file with 16 additions and 16 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,36 +1,36 @@ diff --git a/src/translator/sentence_splitter.cpp b/src/translator/sentence_splitter.cpp index 0370125..9011a69 100644 --- a/src/translator/sentence_splitter.cpp +++ b/src/translator/sentence_splitter.cpp @@ -4,6 +4,8 @@ #include "common/options.h" #include <string> +#include <pcrecpp.h> + namespace marian { namespace bergamot { @@ -30,7 +32,7 @@ SentenceSplitter::SentenceSplitter(marian::Ptr<marian::Options> options) ug::ssplit::SentenceStream SentenceSplitter::createSentenceStream(const string_view &input) { - std::string_view input_converted(input.data(), input.size()); + pcrecpp::StringPiece input_converted(input.data(), input.size()); return std::move( ug::ssplit::SentenceStream(input_converted, this->ssplit_, mode_)); } diff --git a/src/translator/text_processor.cpp b/src/translator/text_processor.cpp index 9d6733e..c3096cc 100644 --- a/src/translator/text_processor.cpp +++ b/src/translator/text_processor.cpp @@ -29,8 +29,7 @@ void TextProcessor::process(const string_view &query, Segments &segments, SentenceRanges &sourceRanges) { auto sentenceStream = sentence_splitter_.createSentenceStream(query); - std::string_view sentenceStringPiece; - + pcrecpp::StringPiece sentenceStringPiece; while (sentenceStream >> sentenceStringPiece) { marian::string_view sentence(sentenceStringPiece.data(), sentenceStringPiece.size()); -
Feb 19, 2021 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,36 @@ diff --git a/src/translator/sentence_splitter.cpp b/src/translator/sentence_splitter.cpp index 9011a69..0370125 100644 --- a/src/translator/sentence_splitter.cpp +++ b/src/translator/sentence_splitter.cpp @@ -4,8 +4,6 @@ #include "common/options.h" #include <string> -#include <pcrecpp.h> - namespace marian { namespace bergamot { @@ -32,7 +30,7 @@ SentenceSplitter::SentenceSplitter(marian::Ptr<marian::Options> options) ug::ssplit::SentenceStream SentenceSplitter::createSentenceStream(const string_view &input) { - pcrecpp::StringPiece input_converted(input.data(), input.size()); + std::string_view input_converted(input.data(), input.size()); return std::move( ug::ssplit::SentenceStream(input_converted, this->ssplit_, mode_)); } diff --git a/src/translator/text_processor.cpp b/src/translator/text_processor.cpp index c3096cc..9d6733e 100644 --- a/src/translator/text_processor.cpp +++ b/src/translator/text_processor.cpp @@ -29,7 +29,8 @@ void TextProcessor::process(const string_view &query, Segments &segments, SentenceRanges &sourceRanges) { auto sentenceStream = sentence_splitter_.createSentenceStream(query); - pcrecpp::StringPiece sentenceStringPiece; + std::string_view sentenceStringPiece; + while (sentenceStream >> sentenceStringPiece) { marian::string_view sentence(sentenceStringPiece.data(), sentenceStringPiece.size());