It looks like this library is not parsing SSE outputs correctly. It assumes that when the stream content is split by \n\ndata:, each line is valid JSON. However, SSE clients should buffer output until they hit a separator character and only then parse the chunk into JSON.
void handle_streamed_response(const std::vector<char>& raw_vec, const bool& is_new_api,
const std::function<void(const std::string& streamed_response)>& stream_callback) {
std::vector<std::string> split_str;
boost::split_regex(split_str, std::string(raw_vec.begin(), raw_vec.end()), boost::regex("\\n\\ndata: *"));
for (auto& s : split_str) {
if (boost::starts_with(s, "data:")) {
s.erase(0, 5);
}
//Remove leading and trailing whitespaces and new lines.
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char c){return !std::isspace(c) && c != '\n';}));
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char c){return !std::isspace(c) && c != '\n';}).base(), s.end());
if (s.empty()) {
continue;
}
if (s == "[DONE]") {
break;
}
try {
nlohmann::json j = nlohmann::json::parse(s);
std::string response;
show original
1 Like