chenx221/Textractor_test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improve JSON parser

Browse Source
This commit is contained in:
Akash Mozumdar 2020-12-15 07:28:12 -07:00
parent 95b145bece
commit 2c9ac1da3c
3 changed files with 35 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/extrawindow.cpp
View File

@ -197,7 +197,7 @@ public:
void AddSentence(QString sentence)
{
if (sentence.size() > maxSentenceSize) sentence = SENTENCE_TOO_BIG;
if (!showOriginal && sentence.count(u8"\x200b \n")) sentence = sentence.split(u8"\x200b \n")[1];
if (!showOriginal && sentence.contains(u8"\x200b \n")) sentence = sentence.split(u8"\x200b \n")[1];
sanitize(sentence);
sentence.chop(std::distance(std::remove(sentence.begin(), sentence.end(), QChar::Tabulation), sentence.end()));
sentenceHistory.push_back(sentence);

2
extensions/network.cpp
View File

@ -61,4 +61,4 @@ std::string Escape(const std::string& text)
return escaped;
}
//TEST(assert(JSON::Parse(LR"([{"string":"hello world","boolean":false,"number": 1.67e+4,"null": null,"array":[]},"hello world"])")))
TEST(assert(JSON::Parse<wchar_t>(LR"([{"string":"hello world","boolean":false,"number":1.67e+4,"null":null,"array":[]},"hello world"])")))

101
extensions/network.h
View File

@ -63,14 +63,52 @@ namespace JSON
};
template <typename C>
std::pair<std::basic_string<C>, int> Unescape(std::basic_string_view<C> text)
struct Value : private std::variant<std::monostate, std::nullopt_t, bool, double, std::basic_string<C>, std::vector<Value<C>>, std::unordered_map<std::basic_string<C>, Value<C>>>
{
using std::variant<std::monostate, std::nullopt_t, bool, double, std::basic_string<C>, std::vector<Value<C>>, std::unordered_map<std::basic_string<C>, Value<C>>>::variant;
explicit operator bool() const { return index(); }
bool IsNull() const { return index() == 1; }
auto Boolean() const { return std::get_if<bool>(this); }
auto Number() const { return std::get_if<double>(this); }
auto String() const { return std::get_if<std::basic_string<C>>(this); }
auto Array() const { return std::get_if<std::vector<Value<C>>>(this); }
auto Object() const { return std::get_if<std::unordered_map<std::basic_string<C>, Value<C>>>(this); }
const Value<C>& operator[](std::basic_string<C> key) const
{
static const Value<C> failure;
if (auto object = Object()) if (auto it = object->find(key); it != object->end()) return it->second;
return failure;
}
const Value<C>& operator[](int i) const
{
static const Value<C> failure;
if (auto array = Array()) if (i < array->size()) return array->at(i);
return failure;
}
};
template <typename C, int maxDepth = 25>
Value<C> Parse(const std::basic_string<C>& text, int64_t& i, int depth)
{
if (depth > maxDepth) return {};
C ch;
auto SkipWhitespace = [&]
{
while (i < text.size() && (text[i] == ' ' || text[i] == '\n' || text[i] == '\r' || text[i] == '\t')) ++i;
if (i >= text.size()) return true;
ch = text[i];
return false;
};
auto ExtractString = [&]
{
std::basic_string<C> unescaped;
int i = 0;
i += 1;
for (; i < text.size(); ++i)
{
auto ch = text[i];
if (ch == '"') return { unescaped, i + 1 };
if (ch == '"') return i += 1, unescaped;
if (ch == '\\')
{
ch = text[i + 1];
@ -91,53 +129,7 @@ namespace JSON
}
else unescaped.push_back(ch);
}
return { unescaped, i };
}
template <typename C>
struct Value : private std::variant<std::monostate, std::nullopt_t, bool, double, std::basic_string<C>, std::vector<Value<C>>, std::unordered_map<std::basic_string<C>, Value<C>>>
{
using std::variant<std::monostate, std::nullopt_t, bool, double, std::basic_string<C>, std::vector<Value<C>>, std::unordered_map<std::basic_string<C>, Value<C>>>::variant;
explicit operator bool() const { return index(); }
bool IsNull() const { return index() == 1; }
auto Boolean() const { return std::get_if<bool>(this); }
auto String() const { return std::get_if<std::basic_string<C>>(this); }
auto Array() const { return std::get_if<std::vector<Value<C>>>(this); }
auto Object() const { return std::get_if<std::unordered_map<std::basic_string<C>, Value<C>>>(this); }
const Value<C>& operator[](std::basic_string<C> key) const
{
static const Value<C> failure;
if (auto object = Object()) if (auto it = object->find(key); it != object->end()) return it->second;
return failure;
}
const Value<C>& operator[](int i) const
{
static const Value<C> failure;
if (auto array = Array()) if (i < array->size()) return array->at(i);
return failure;
}
};
template <typename C, int maxDepth = 25>
Value<C> Parse(std::basic_string_view<C> text, int64_t& i, int depth)
{
if (depth > maxDepth) return {};
C ch;
auto SkipWhitespace = [&]
{
while (i < text.size() && (text[i] == ' ' || text[i] == '\n' || text[i] == '\r' || text[i] == '\t')) ++i;
if (i >= text.size()) return true;
ch = text[i];
return false;
};
auto ExtractString = [&]
{
i += 1;
auto [string, length] = Unescape(text.substr(i));
i += length;
return string;
return unescaped;
};
if (SkipWhitespace()) return {};
@ -155,9 +147,10 @@ namespace JSON
if (ch == '-' || (ch >= '0' && ch <= '9'))
{
// no numbers currently used, add an actual parser when needed
while (i < text.size() && ((text[i] >= '0' && text[i] <= '9') || text[i] == '-' || text[i] == '+' || text[i] == 'e' || text[i] == 'E' || text[i] == '.')) ++i;
return 0.0;
std::string number;
for (; i < text.size() && ((text[i] >= '0' && text[i] <= '9') || text[i] == '-' || text[i] == '+' || text[i] == 'e' || text[i] == 'E' || text[i] == '.'); ++i)
number.push_back(text[i]);
return strtod(number.c_str(), NULL);
}
if (ch == '"') return ExtractString();
@ -203,6 +196,6 @@ namespace JSON
Value<C> Parse(const std::basic_string<C>& text)
{
int64_t start = 0;
return Parse((std::basic_string_view<C>)text, start, 0);
return Parse(text, start, 0);
}
}