diff options
Diffstat (limited to 'src/mpc-hc/SubtitlesProvider.cpp')
| -rw-r--r-- | src/mpc-hc/SubtitlesProvider.cpp | 1127 |
1 files changed, 1127 insertions, 0 deletions
diff --git a/src/mpc-hc/SubtitlesProvider.cpp b/src/mpc-hc/SubtitlesProvider.cpp new file mode 100644 index 000000000..d5075f788 --- /dev/null +++ b/src/mpc-hc/SubtitlesProvider.cpp @@ -0,0 +1,1127 @@ +/* + * (C) 2016-2017 see Authors.txt + * + * This file is part of MPC-HC. + * + * MPC-HC is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * MPC-HC is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include "stdafx.h" +#include "SubtitlesProvider.h" +#include "SubtitlesProvidersUtils.h" +#include "mplayerc.h" +#include "ISOLang.h" +#include "Logger.h" +#include "MediaInfo/library/Source/ThirdParty/base64/base64.h" +#include "tinyxml2/library/tinyxml2.h" +#include "rapidjson/include/rapidjson/document.h" + +#define LOG if (AfxGetAppSettings().bEnableLogging) SUBTITLES_LOG +#define LOG_NONE _T("()") +#define LOG_INPUT _T("(\"%S\")") +#define LOG_OUTPUT _T("()=%S") +#define LOG_BOTH _T("(\"%S\")=%S") +#define LOG_ERROR _T("() ERROR: %S") + +#define GUESSED_NAME_POSTFIX " (*)" +#define CheckAbortAndReturn() { if (IsAborting()) return SR_ABORTED; } + +using namespace SubtitlesProvidersUtils; + +class LanguageDownloadException : public std::exception +{ + using exception::exception; +}; + +/****************************************************************************** +** Register providers +******************************************************************************/ +void SubtitlesProviders::RegisterProviders() +{ + Register<OpenSubtitles>(this); + Register<podnapisi>(this); + Register<titlovi>(this); + Register<SubDB>(this); + Register<ysubs>(this); + Register<Napisy24>(this); +} + +/****************************************************************************** +** OpenSubtitles +******************************************************************************/ + +void OpenSubtitles::Initialize() +{ + xmlrpc = std::make_unique<XmlRpcClient>("http://api.opensubtitles.org/xml-rpc"); + xmlrpc->setIgnoreCertificateAuthority(); +} + +SRESULT OpenSubtitles::Login(const std::string& sUserName, const std::string& sPassword) +{ + if (xmlrpc) { + XmlRpcValue args, result; + args[0] = sUserName; + args[1] = sPassword; + args[2] = "en"; + const auto& strUA = UserAgent(); + args[3] = strUA.c_str(); // Test with "OSTestUserAgent" + if (!xmlrpc->execute("LogIn", args, result)) { + return SR_FAILED; + } + + if (result["status"].getType() == XmlRpcValue::Type::TypeString) { + if (result["status"] == std::string("200 OK")) { + token = result["token"]; + } else if (result["status"] == std::string("401 Unauthorized")) { + // Notify user that User/Pass provided are invalid. + CString msg; + msg.Format(IDS_SUB_CREDENTIALS_ERROR, Name().c_str(), UserName().c_str()); + AfxMessageBox(msg, MB_ICONERROR | MB_OK); + } + } + } + + LOG(LOG_BOTH, sUserName.c_str(), token.valid() ? (LPCSTR)token : "failed"); + return token.valid() ? SR_SUCCEEDED : SR_FAILED; +} + +SRESULT OpenSubtitles::LogOut() +{ + if (xmlrpc && token.valid()) { + XmlRpcValue args, result; + args[0] = token; + VERIFY(xmlrpc->execute("LogOut", args, result)); + token.clear(); + } + m_nLoggedIn = SPL_UNDEFINED; + + LOG(LOG_NONE); + return SR_SUCCEEDED; +} + +SRESULT OpenSubtitles::Hash(SubtitlesInfo& pFileInfo) +{ + pFileInfo.fileHash = StringFormat("%016I64x", GenerateOSHash(pFileInfo)); + LOG(LOG_OUTPUT, pFileInfo.fileHash.c_str()); + return SR_SUCCEEDED; +} + +SRESULT OpenSubtitles::Search(const SubtitlesInfo& pFileInfo) +{ + const auto languages = LanguagesISO6392(); + XmlRpcValue args, result; + args[0] = token; + auto& movieInfo = args[1][0]; + movieInfo["sublanguageid"] = !languages.empty() ? JoinContainer(languages, ",") : "all"; + movieInfo["moviehash"] = pFileInfo.fileHash; + movieInfo["moviebytesize"] = std::to_string(pFileInfo.fileSize); + //args[1][1]["sublanguageid"] = !languages.empty() ? languages : "all"; + //args[1][1]["tag"] = pFileInfo.fileName + "." + pFileInfo.fileExtension; + args[2]["limit"] = 500; + + LOG(LOG_INPUT, + StringFormat("{ sublanguageid=\"%s\", moviehash=\"%s\", moviebytesize=\"%s\", limit=%d }", + (LPCSTR)movieInfo["sublanguageid"], + (LPCSTR)movieInfo["moviehash"], + (LPCSTR)movieInfo["moviebytesize"], + (int)args[2]["limit"]).c_str()); + + if (!xmlrpc->execute("SearchSubtitles", args, result)) { + return SR_FAILED; + } + + if (result["data"].getType() != XmlRpcValue::Type::TypeArray) { + return SR_FAILED; + } + + int nCount = result["data"].size(); + for (int i = 0; i < nCount; ++i) { + CheckAbortAndReturn(); + XmlRpcValue& data(result["data"][i]); + SubtitlesInfo pSubtitlesInfo; + pSubtitlesInfo.id = (const char*)data["IDSubtitleFile"]; + pSubtitlesInfo.discNumber = data["SubActualCD"]; + pSubtitlesInfo.discCount = data["SubSumCD"]; + pSubtitlesInfo.fileExtension = (const char*)data["SubFormat"]; + pSubtitlesInfo.languageCode = (const char*)data["ISO639"]; //"SubLanguageID" + pSubtitlesInfo.languageName = (const char*)data["LanguageName"]; + pSubtitlesInfo.downloadCount = data["SubDownloadsCnt"]; + + pSubtitlesInfo.fileName = (const char*)data["SubFileName"]; + regexResult results; + stringMatch("\"([^\"]+)\" (.+)", (const char*)data["MovieName"], results); + if (!results.empty()) { + pSubtitlesInfo.title = results[0]; + pSubtitlesInfo.title2 = results[1]; + } else { + pSubtitlesInfo.title = (const char*)data["MovieName"]; + } + pSubtitlesInfo.year = (int)data["MovieYear"] == 0 ? -1 : (int)data["MovieYear"]; + pSubtitlesInfo.seasonNumber = (int)data["SeriesSeason"] == 0 ? -1 : (int)data["SeriesSeason"]; + pSubtitlesInfo.episodeNumber = (int)data["SeriesEpisode"] == 0 ? -1 : (int)data["SeriesEpisode"]; + pSubtitlesInfo.hearingImpaired = data["SubHearingImpaired"]; + pSubtitlesInfo.url = (const char*)data["SubtitlesLink"]; + pSubtitlesInfo.releaseNames.emplace_back((const char*)data["MovieReleaseName"]); + pSubtitlesInfo.imdbid = (const char*)data["IDMovieImdb"]; + pSubtitlesInfo.corrected = (int)data["SubBad"] ? -1 : 0; + Set(pSubtitlesInfo); + } + return SR_SUCCEEDED; +} + +SRESULT OpenSubtitles::Download(SubtitlesInfo& pSubtitlesInfo) +{ + XmlRpcValue args, result; + args[0] = token; + args[1][0] = pSubtitlesInfo.id; + if (!xmlrpc->execute("DownloadSubtitles", args, result)) { + return SR_FAILED; + } + + LOG(LOG_INPUT, pSubtitlesInfo.id.c_str()); + + if (result["data"].getType() != XmlRpcValue::Type::TypeArray) { + return SR_FAILED; + } + + pSubtitlesInfo.fileContents = Base64::decode(std::string(result["data"][0]["data"])); + return SR_SUCCEEDED; +} + +SRESULT OpenSubtitles::Upload(const SubtitlesInfo& pSubtitlesInfo) +{ + XmlRpcValue args, result; + args[0] = token; + + //TODO: Ask how to obtain commented values !!! + args[1]["cd1"]["subhash"] = StringToHash(pSubtitlesInfo.fileContents, CALG_MD5); + args[1]["cd1"]["subfilename"] = pSubtitlesInfo.fileName + ".srt"; + args[1]["cd1"]["moviehash"] = pSubtitlesInfo.fileHash; + args[1]["cd1"]["moviebytesize"] = std::to_string(pSubtitlesInfo.fileSize); + //args[1]["cd1"]["movietimems"]; + //args[1]["cd1"]["movieframes"]; + //args[1]["cd1"]["moviefps"]; + args[1]["cd1"]["moviefilename"] = pSubtitlesInfo.fileName + "." + pSubtitlesInfo.fileExtension; + + CheckAbortAndReturn(); + if (!xmlrpc->execute("TryUploadSubtitles", args, result)) { + return SR_FAILED; + } + CheckAbortAndReturn(); + + if ((int)result["alreadyindb"] == 1) { + return SR_EXISTS; + } else if ((int)result["alreadyindb"] == 0) { + // We need imdbid to proceed + if (result["data"].getType() == XmlRpcValue::Type::TypeArray) { + args[1]["baseinfo"]["idmovieimdb"] = result["data"][0]["IDMovieImdb"]; + } else if (!pSubtitlesInfo.imdbid.empty()) { + args[1]["baseinfo"]["idmovieimdb"] = pSubtitlesInfo.imdbid; + } else { + std::string title(StringReplace(pSubtitlesInfo.title, "and", "&")); + if (!args[1]["baseinfo"]["idmovieimdb"].valid()) { + XmlRpcValue _args, _result; + _args[0] = token; + _args[1][0] = pSubtitlesInfo.fileHash; + if (!xmlrpc->execute("CheckMovieHash", _args, _result)) { + return SR_FAILED; + } + + if (_result["data"].getType() == XmlRpcValue::Type::TypeStruct) { + //regexResults results; + //stringMatch("\"(.+)\" (.+)", (const char*)data["MovieName"], results); + //if (!results.empty()) { + // pSubtitlesInfo.title = results[0][0]; + // pSubtitlesInfo.title2 = results[0][1]; + //} else { + // pSubtitlesInfo.title = (const char*)data["MovieName"]; + //} + regexResults results; + stringMatch("\"(.+)\" .+|(.+)", StringReplace((const char*)_result["data"][pSubtitlesInfo.fileHash]["MovieName"], "and", "&"), results); + std::string _title(results[0][0] + results[0][1]); + + if (_stricmp(title.c_str(), _title.c_str()) == 0 /*&& (pSubtitlesInfo.year == -1 || (pSubtitlesInfo.year != -1 && pSubtitlesInfo.year == atoi(_result["data"][pSubtitlesInfo.fileHash]["MovieYear"])))*/) { + args[1]["baseinfo"]["idmovieimdb"] = _result["data"][pSubtitlesInfo.fileHash]["MovieImdbID"]; //imdbid + } + } + } + + if (!args[1]["baseinfo"]["idmovieimdb"].valid()) { + XmlRpcValue _args, _result; + _args[0] = token; + _args[1][0] = pSubtitlesInfo.fileHash; + if (!xmlrpc->execute("CheckMovieHash2", _args, _result)) { + return SR_FAILED; + } + + if (_result["data"].getType() == XmlRpcValue::Type::TypeArray) { + int nCount = _result["data"][pSubtitlesInfo.fileHash].size(); + for (int i = 0; i < nCount; ++i) { + regexResults results; + stringMatch("\"(.+)\" .+|(.+)", StringReplace((const char*)_result["data"][pSubtitlesInfo.fileHash][i]["MovieName"], "and", "&"), results); + std::string _title(results[0][0] + results[0][1]); + + if (_stricmp(title.c_str(), _title.c_str()) == 0 /*&& (pSubtitlesInfo.year == -1 || (pSubtitlesInfo.year != -1 && pSubtitlesInfo.year == atoi(_result["data"][pSubtitlesInfo.fileHash][i]["MovieYear"])))*/) { + args[1]["baseinfo"]["idmovieimdb"] = _result["data"][pSubtitlesInfo.fileHash][i]["MovieImdbID"]; //imdbid + break; + } + } + } + } + + if (!args[1]["baseinfo"]["idmovieimdb"].valid()) { + XmlRpcValue _args, _result; + _args[0] = token; + _args[1] = title; + if (!xmlrpc->execute("SearchMoviesOnIMDB", _args, _result)) { + return SR_FAILED; + } + if (_result["data"].getType() == XmlRpcValue::Type::TypeArray) { + int nCount = _result["data"].size(); + for (int i = 0; i < nCount; ++i) { + regexResults results; + stringMatch("(.+) [(](\\d{4})[)]", StringReplace((const char*)_result["data"][i]["title"], "and", "&"), results); + if (results.size() == 1) { + std::string _title(results[0][0]); + + if (_stricmp(title.c_str(), _title.c_str()) == 0 /*&& (pSubtitlesInfo.year == -1 || (pSubtitlesInfo.year != -1 && pSubtitlesInfo.year == atoi(results[0][1].c_str())))*/) { + args[1]["baseinfo"]["idmovieimdb"] = _result["data"][i]["id"]; //imdbid + break; + } + } + } + } + } + } + if (args[1]["baseinfo"]["idmovieimdb"].valid()) { + XmlRpcValue _args, _result; + _args[0] = token; + _args[1][0]["moviehash"] = pSubtitlesInfo.fileHash; + _args[1][0]["moviebytesize"] = (int)pSubtitlesInfo.fileSize; + _args[1][0]["imdbid"] = args[1]["baseinfo"]["idmovieimdb"]; + //_args[1][0]["movietimems"]; + //_args[1][0]["moviefps"]; + _args[1][0]["moviefilename"] = pSubtitlesInfo.fileName + "." + pSubtitlesInfo.fileExtension; + if (!xmlrpc->execute("InsertMovieHash", _args, _result)) { + return SR_FAILED; + } + // REsult value is irrelevant + _result["data"]["accepted_moviehashes"]; + + + //args[1]["baseinfo"]["moviereleasename"]; + //args[1]["baseinfo"]["movieaka"]; + //args[1]["baseinfo"]["sublanguageid"]; + //args[1]["baseinfo"]["subauthorcomment"]; + if (pSubtitlesInfo.hearingImpaired != -1) { + args[1]["baseinfo"]["hearingimpaired"] = pSubtitlesInfo.hearingImpaired; + } + //args[1]["baseinfo"]["highdefinition"]; + //args[1]["baseinfo"]["automatictranslation"]; + + args[1]["cd1"]["subcontent"] = Base64::encode(StringGzipCompress(pSubtitlesInfo.fileContents)); + + if (!xmlrpc->execute("UploadSubtitles", args, result)) { + return SR_FAILED; + } + LOG(LOG_OUTPUT, (LPCSTR)result["data"]); + + return SR_SUCCEEDED; + } + } + return SR_FAILED; +} + +const std::set<std::string>& OpenSubtitles::Languages() const +{ + static std::once_flag initialized; + static std::set<std::string> result; + + try { + std::call_once(initialized, [this]() { + if (!CheckInternetConnection()) { + throw LanguageDownloadException("No internet connection."); + } + XmlRpcValue args, res; + args = "en"; + if (!xmlrpc->execute("GetSubLanguages", args, res)) { + throw LanguageDownloadException("Failed to execute xmlrpc command."); + } + if (res["data"].getType() != XmlRpcValue::Type::TypeArray) { + throw LanguageDownloadException("Response is not an array."); + } + + auto& data = res["data"]; + int count = data.size(); + for (int i = 0; i < count; ++i) { +#ifdef _DEBUG + // Validate if language code conversion is in sync with OpenSubtitles database. + std::string subLanguageID = data[i]["SubLanguageID"]; + std::string ISO6391 = data[i]["ISO639"]; + ASSERT(!ISO6391.empty()); + ASSERT(!subLanguageID.empty()); + ASSERT(ISOLang::ISO6391To6392(ISO6391.c_str()) == subLanguageID.c_str()); + ASSERT(ISOLang::ISO6392To6391(subLanguageID.c_str()) == ISO6391.c_str()); + //std::string languageName = data[i]["LanguageName"]; + //ASSERT(ISO639XToLanguage(ISO6391.c_str()) == languageName.c_str()); + //ASSERT(ISO639XToLanguage(subLanguageID.c_str()) == languageName.c_str()); +#endif + result.emplace(data[i]["ISO639"]); + } + }); + } catch (const LanguageDownloadException& e) { + UNREFERENCED_PARAMETER(e); + LOG(LOG_ERROR, e.what()); + } + return result; +} + +bool OpenSubtitles::NeedLogin() +{ + // return true to call Login() or false to skip Login() + if (!token.valid()) { + return true; + } + + XmlRpcValue args, result; + args[0] = token; + if (!xmlrpc->execute("NoOperation", args, result)) { + return false; + } + + if ((result["status"].getType() == XmlRpcValue::Type::TypeString) && (result["status"] == std::string("200 OK"))) { + return false; + } + + return true; +} + + +/****************************************************************************** +** SubDB +******************************************************************************/ + +SRESULT SubDB::Hash(SubtitlesInfo& pFileInfo) +{ + std::vector<BYTE> buffer(2 * PROBE_SIZE); + if (pFileInfo.pAsyncReader) { + UINT64 position = 0; + pFileInfo.pAsyncReader->SyncRead(position, PROBE_SIZE, (BYTE*)&buffer[0]); + position = std::max((UINT64)0, (UINT64)(pFileInfo.fileSize - PROBE_SIZE)); + pFileInfo.pAsyncReader->SyncRead(position, PROBE_SIZE, (BYTE*)&buffer[PROBE_SIZE]); + } else { + CFile file; + CFileException fileException; + if (file.Open(CString(pFileInfo.filePath.c_str()), + CFile::modeRead | CFile::osSequentialScan | CFile::shareDenyNone | CFile::typeBinary, + &fileException)) { + file.Read(&buffer[0], PROBE_SIZE); + file.Seek(std::max((UINT64)0, (UINT64)(pFileInfo.fileSize - PROBE_SIZE)), CFile::begin); + file.Read(&buffer[PROBE_SIZE], PROBE_SIZE); + } + } + pFileInfo.fileHash = StringToHash(std::string((char*)&buffer[0], buffer.size()), CALG_MD5); + LOG(LOG_OUTPUT, pFileInfo.fileHash.c_str()); + return SR_SUCCEEDED; +} + +SRESULT SubDB::Search(const SubtitlesInfo& pFileInfo) +{ + SRESULT searchResult = SR_UNDEFINED; + std::string url(StringFormat("http://api.thesubdb.com/?action=search&hash=%s", pFileInfo.fileHash.c_str())); + LOG(LOG_INPUT, url.c_str()); + + std::string data; + searchResult = DownloadInternal(url, "", data); + + if (!data.empty()) { + for (const auto& iter : StringTokenize(data, ",")) { + CheckAbortAndReturn(); + if (CheckLanguage(iter)) { + SubtitlesInfo pSubtitlesInfo; + pSubtitlesInfo.id = pFileInfo.fileHash; + pSubtitlesInfo.fileExtension = "srt"; + pSubtitlesInfo.fileName = pFileInfo.fileName + GUESSED_NAME_POSTFIX; + pSubtitlesInfo.languageCode = iter; + pSubtitlesInfo.languageName = UTF16To8(ISOLang::ISO639XToLanguage(iter.c_str())); + pSubtitlesInfo.discNumber = 1; + pSubtitlesInfo.discCount = 1; + pSubtitlesInfo.title = pFileInfo.title; + Set(pSubtitlesInfo); + } + } + } + + return searchResult; +} + +SRESULT SubDB::Download(SubtitlesInfo& pSubtitlesInfo) +{ + std::string url(StringFormat("http://api.thesubdb.com/?action=download&hash=%s&language=%s", pSubtitlesInfo.id.c_str(), pSubtitlesInfo.languageCode.c_str())); + LOG(LOG_INPUT, url.c_str()); + return DownloadInternal(url, "", pSubtitlesInfo.fileContents); +} + +SRESULT SubDB::Upload(const SubtitlesInfo& pSubtitlesInfo) +{ +#define MULTIPART_BOUNDARY "xYzZY" + std::string url(StringFormat("http://api.thesubdb.com/?action=upload&hash=%s", pSubtitlesInfo.fileHash.c_str())); + stringMap headers({ + { "User-Agent", UserAgent() }, + { "Content-Type", "multipart/form-data; boundary=" MULTIPART_BOUNDARY }, + }); + + std::string content, data; + content += StringFormat("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", MULTIPART_BOUNDARY, "hash", pSubtitlesInfo.fileHash.c_str()); + content += StringFormat("--%s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"%s.%s\"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n", + MULTIPART_BOUNDARY, "file", pSubtitlesInfo.fileHash.c_str(), "srt"); + content += pSubtitlesInfo.fileContents; + content += StringFormat("\r\n--%s--\r\n", MULTIPART_BOUNDARY); + + CheckAbortAndReturn(); + + DWORD dwStatusCode = NULL; + StringUpload(url, headers, content, data, FALSE, &dwStatusCode); + + LOG(LOG_BOTH, url.c_str(), std::to_string(dwStatusCode).c_str()); + + switch (dwStatusCode) { + case 201: + return SR_SUCCEEDED; //'Uploaded': (HTTP/1.1 201 Created): If everything was OK, the HTTP status code 201 will be returned. + case 403: + return SR_EXISTS; //'Duplicated': (HTTP/1.1 403 Forbidden): If the subtitle file already exists in our database, the HTTP status code 403 will be returned. + case 400: + return SR_FAILED; //'Malformed': (HTTP/1.1 400 Bad Request): If the request was malformed, the HTTP status code 400 will be returned. + case 415: + return SR_FAILED; //'Invalid': (HTTP/1.1 415 Unsupported Media Type): If the subtitle file is not supported by our database, the HTTP status code 415 will be returned. + default: + return SR_UNDEFINED; + } +} + +const std::set<std::string>& SubDB::Languages() const +{ + static std::once_flag initialized; + static std::set<std::string> result; + try { + std::call_once(initialized, [this]() { + if (!CheckInternetConnection()) { + throw LanguageDownloadException("No internet connection."); + } + std::string data; + if (DownloadInternal("http://api.thesubdb.com/?action=languages", "", data) != SR_SUCCEEDED) { + throw LanguageDownloadException("Failed to download language list."); + } + for (const auto& str : StringTokenize(data, ",")) { + result.emplace(str); + } + }); + } catch (const LanguageDownloadException& e) { + UNREFERENCED_PARAMETER(e); + LOG(LOG_ERROR, e.what()); + } + return result; +} + +/****************************************************************************** +** podnapisi +******************************************************************************/ + +SRESULT podnapisi::Login(const std::string& sUserName, const std::string& sPassword) +{ + //TODO: implement + return SR_UNDEFINED; +} + +/* +UPDATED +https://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652 +RESULTS ------------------------------------------------ +"/sXML/1/" //Reply in XML format +"/page//" //Return nth page of results +SEARCH ------------------------------------------------- +"/sT/1/" //Type: -1=all, 0=movies, 1=series, don't specify for auto detection +"/sAKA/1/" //Include movie title aliases +"/sM//" //Movie id from www.omdb.si +"/sK//" //Title url encoded text +"/sY//" //Year number +"/sTS//" //Season number +"/sTE//" //Episode number +"/sR//" //Release name url encoded text +"/sJ/0/" //Languages (old integer IDs), comma delimited, 0=all +"/sL/en/" //Languages in ISO ISO codes (exception are sr-latn and pt-br), comma delimited +"/sEH//" //Exact hash match (OSH) +"/sMH//" //Movie hash (OSH) +SEARCH ADDITIONAL -------------------------------------- +"/sFT/0/" //Subtitles Format: 0=all, 1=MicroDVD, 2=SAMI, 3=SSA, 4=SubRip, 5=SubViewer 2.0, 6=SubViewer, 7=MPSub, 8=Advanced SSA, 9=DVDSubtitle, 10=TMPlayer, 11=MPlayer2 +"/sA/0/" //Search subtitles by user id, 0=all +"/sI//" //Search subtitles by subtitle id +SORTING ------------------------------------------------ +"/sS//" //Sorting field: movie, year, fps, language, downloads, cds, username, time, rating +"/sO//" //Soring order: asc, desc +FILTERS ------------------------------------------------ +"/sOE/1/" //Subtitles for extended edition only +"/sOD/1/" //Subtitles suitable for DVD only +"/sOH/1/" //Subtitles for high-definition video only +"/sOI/1/" //Subtitles for hearing impaired only +"/sOT/1/" //Technically correct only +"/sOL/1/" //Grammatically correct only +"/sOA/1/" //Author subtitles only +"/sOCS/1/" //Only subtitles for a complete season +UNKNOWN ------------------------------------------------ +"/sH//" //Search subtitles by video file hash ??? (not working for me) +*/ + +SRESULT podnapisi::Search(const SubtitlesInfo& pFileInfo) +{ + SRESULT searchResult = SR_UNDEFINED; + int page = 1, pages = 1, results = 0; + do { + CheckAbortAndReturn(); + + const auto languages = LanguagesISO6391(); + std::string search(pFileInfo.title); + if (!pFileInfo.country.empty()) { + search += " " + pFileInfo.country; + } + search = std::regex_replace(search, std::regex(" and | *[!?&':] *", RegexFlags), " "); + + std::string url("https://www.podnapisi.net/ppodnapisi/search"); + url += "?sXML=1"; + url += "&sAKA=1"; + url += (!search.empty() ? "&sK=" + UrlEncode(search.c_str()) : ""); + url += (pFileInfo.year != -1 ? "&sY=" + std::to_string(pFileInfo.year) : ""); + url += (pFileInfo.seasonNumber != -1 ? "&sTS=" + std::to_string(pFileInfo.seasonNumber) : ""); + url += (pFileInfo.episodeNumber != -1 ? "&sTE=" + std::to_string(pFileInfo.episodeNumber) : ""); + url += "&sMH=" + pFileInfo.fileHash; + //url += "&sR=" + UrlEncode(pFileInfo.fileName.c_str()); + url += (!languages.empty() ? "&sL=" + JoinContainer(languages, ",") : ""); + url += "&page=" + std::to_string(page); + LOG(LOG_INPUT, url.c_str()); + + std::string data; + searchResult = DownloadInternal(url, "", data); + + using namespace tinyxml2; + + tinyxml2::XMLDocument dxml; + if (dxml.Parse(data.c_str()) == XML_SUCCESS) { + + auto GetChildElementText = [&](XMLElement * pElement, const char* value) -> std::string { + std::string str; + XMLElement* pChildElement = pElement->FirstChildElement(value); + if (pChildElement != nullptr) + { + auto pText = pChildElement->GetText(); + if (pText != nullptr) { + str = pText; + } + } + return str; + }; + + XMLElement* pRootElmt = dxml.FirstChildElement("results"); + if (pRootElmt) { + XMLElement* pPaginationElmt = pRootElmt->FirstChildElement("pagination"); + if (pPaginationElmt) { + page = atoi(GetChildElementText(pPaginationElmt, "current").c_str()); + pages = atoi(GetChildElementText(pPaginationElmt, "count").c_str()); + results = atoi(GetChildElementText(pPaginationElmt, "results").c_str()); + } + // 30 results per page + if (page > 1) { + return SR_TOOMANY; + } + + if (results > 0) { + XMLElement* pSubtitleElmt = pRootElmt->FirstChildElement("subtitle"); + + while (pSubtitleElmt) { + CheckAbortAndReturn(); + + SubtitlesInfo pSubtitlesInfo; + + pSubtitlesInfo.id = GetChildElementText(pSubtitleElmt, "pid"); + pSubtitlesInfo.title = HtmlSpecialCharsDecode(GetChildElementText(pSubtitleElmt, "title").c_str()); + + std::string year = GetChildElementText(pSubtitleElmt, "year"); + pSubtitlesInfo.year = year.empty() ? -1 : atoi(year.c_str()); + + pSubtitlesInfo.url = GetChildElementText(pSubtitleElmt, "url"); + std::string format = GetChildElementText(pSubtitleElmt, "format"); + pSubtitlesInfo.fileExtension = (format == "SubRip" || format == "N/A") ? "srt" : format; + + pSubtitlesInfo.languageCode = podnapisi_languages[atoi(GetChildElementText(pSubtitleElmt, "languageId").c_str())].code; + pSubtitlesInfo.languageName = GetChildElementText(pSubtitleElmt, "languageName"); + pSubtitlesInfo.seasonNumber = atoi(GetChildElementText(pSubtitleElmt, "tvSeason").c_str()); + pSubtitlesInfo.episodeNumber = atoi(GetChildElementText(pSubtitleElmt, "tvEpisode").c_str()); + pSubtitlesInfo.discCount = atoi(GetChildElementText(pSubtitleElmt, "cds").c_str()); + pSubtitlesInfo.discNumber = pSubtitlesInfo.discCount; + + std::string flags = GetChildElementText(pSubtitleElmt, "flags"); + pSubtitlesInfo.hearingImpaired = (flags.find("n") != std::string::npos) ? TRUE : FALSE; + pSubtitlesInfo.corrected = (flags.find("r") != std::string::npos) ? -1 : 0; + pSubtitlesInfo.downloadCount = atoi(GetChildElementText(pSubtitleElmt, "downloads").c_str()); + pSubtitlesInfo.imdbid = GetChildElementText(pSubtitleElmt, "movieId"); + pSubtitlesInfo.frameRate = atof(GetChildElementText(pSubtitleElmt, "fps").c_str()); + + XMLElement* pReleasesElem = pSubtitleElmt->FirstChildElement("releases"); + if (pReleasesElem) { + XMLElement* pReleaseElem = pReleasesElem->FirstChildElement("release"); + + while (pReleaseElem) { + auto pText = pReleaseElem->GetText(); + + if (!pText) { + continue; + } + + pSubtitlesInfo.releaseNames.emplace_back(pText); + + if (pSubtitlesInfo.fileName.empty() || pFileInfo.fileName.find(pText) != std::string::npos) { + pSubtitlesInfo.fileName = pText; + pSubtitlesInfo.fileName += "." + pSubtitlesInfo.fileExtension; + } + pReleaseElem = pReleaseElem->NextSiblingElement(); + } + } + + if (pSubtitlesInfo.fileName.empty()) { + std::string str = pSubtitlesInfo.title; + if (!year.empty()) { + str += " " + year; + } + if (pSubtitlesInfo.seasonNumber > 0) { + str += StringFormat(" S%02d", pSubtitlesInfo.seasonNumber); + } + if (pSubtitlesInfo.episodeNumber > 0) { + str += StringFormat("%sE%02d", (pSubtitlesInfo.seasonNumber > 0) ? "" : " ", pSubtitlesInfo.episodeNumber); + } + str += GUESSED_NAME_POSTFIX; + pSubtitlesInfo.fileName = str; + } + + Set(pSubtitlesInfo); + pSubtitleElmt = pSubtitleElmt->NextSiblingElement(); + } + } + } + } + } while (page++ < pages); + + return searchResult; +} + +SRESULT podnapisi::Hash(SubtitlesInfo& pFileInfo) +{ + pFileInfo.fileHash = StringFormat("%016I64x", GenerateOSHash(pFileInfo)); + LOG(LOG_OUTPUT, pFileInfo.fileHash.c_str()); + return SR_SUCCEEDED; +} + +SRESULT podnapisi::Download(SubtitlesInfo& pSubtitlesInfo) +{ + std::string url = StringFormat("https://www.podnapisi.net/subtitles/%s/download", pSubtitlesInfo.id.c_str()); + LOG(LOG_INPUT, url.c_str()); + return DownloadInternal(url, "", pSubtitlesInfo.fileContents); +} + +const std::set<std::string>& podnapisi::Languages() const +{ + static std::once_flag initialized; + static std::set<std::string> result; + + std::call_once(initialized, [this]() { + for (const auto& iter : podnapisi_languages) { + if (strlen(iter.code)) { + result.emplace(iter.code); + } + } + }); + return result; +} + +/****************************************************************************** +** titlovi +******************************************************************************/ + +/* + x-dev_api_id= + uiculture=hr,rs,si,ba,en,mk + language=hr,rs,sr,si,ba,en,mk + keyword= + year= + mt=numeric value representing type of subtitle (Movie / TV show / documentary 1, 2, 3) + season=numeric value representing season + episode=numeric value representing season episode + forcefilename=true (default is false) return direct download link +*/ + +SRESULT titlovi::Search(const SubtitlesInfo& pFileInfo) +{ + // Need to filter not supported languages, because their API returns .hr language otherwise. + auto selectedLanguages = LanguagesISO6391(); + bool userSelectedLanguage = !selectedLanguages.empty(); + const auto languagesIntersection = GetLanguagesIntersection(std::move(selectedLanguages)); + std::string KEY = "WC1ERVYtREVTS1RPUF9maWUyYS1hMVJzYS1hSHc0UA=="; + std::string url(StringFormat("http://api.titlovi.com/xml_get_api.ashx?x-dev_api_id=%s&uiculture=en&forcefilename=true", Base64::decode(KEY).c_str())); + url += "&mt=" + (pFileInfo.seasonNumber != -1 ? std::to_string(2) : std::to_string(1)); + url += "&keyword=" + UrlEncode(pFileInfo.title.c_str()); + url += (pFileInfo.seasonNumber != -1 ? "&season=" + std::to_string(pFileInfo.seasonNumber) : ""); + url += (pFileInfo.episodeNumber != -1 ? "&episode=" + std::to_string(pFileInfo.episodeNumber) : ""); + url += (pFileInfo.year != -1 ? "&year=" + std::to_string(pFileInfo.year) : ""); + url += (userSelectedLanguage ? "&language=" + JoinContainer(languagesIntersection, ",") : ""); + LOG(LOG_INPUT, url.c_str()); + + std::string data; + SRESULT searchResult = DownloadInternal(url, "", data); + + tinyxml2::XMLDocument dxml; + if (dxml.Parse(data.c_str()) == tinyxml2::XMLError::XML_SUCCESS) { + + auto GetChildElementText = [&](tinyxml2::XMLElement * pElement, const char* value) -> std::string { + std::string str; + auto pChildElement = pElement->FirstChildElement(value); + + if (pChildElement != nullptr) + { + auto pText = pChildElement->GetText(); + if (pText != nullptr) { + str = pText; + } + } + return str; + }; + + auto pRootElmt = dxml.FirstChildElement("subtitles"); + + if (pRootElmt) { + int num = pRootElmt->IntAttribute("resultsCount"); + + if (num > 0/* && num < 50*/) { + auto pSubtitleElmt = pRootElmt->FirstChildElement(); + + while (pSubtitleElmt) { + SubtitlesInfo pSubtitlesInfo; + + pSubtitlesInfo.title = GetChildElementText(pSubtitleElmt, "title"); + pSubtitlesInfo.languageCode = GetChildElementText(pSubtitleElmt, "language"); + + for (const auto& language : titlovi_languages) { + if (pSubtitlesInfo.languageCode == language.code) { + pSubtitlesInfo.languageCode = language.name; + } + } + + pSubtitlesInfo.languageName = UTF16To8(ISOLang::ISO639XToLanguage(pSubtitlesInfo.languageCode.c_str())); + auto releaseNames = StringTokenize(GetChildElementText(pSubtitleElmt, "release"), "/"); + pSubtitlesInfo.releaseNames = { releaseNames.begin(), releaseNames.end() }; + pSubtitlesInfo.imdbid = GetChildElementText(pSubtitleElmt, "imdbId"); + pSubtitlesInfo.frameRate = atof(GetChildElementText(pSubtitleElmt, "fps").c_str()); + pSubtitlesInfo.year = atoi(GetChildElementText(pSubtitleElmt, "year").c_str()); + pSubtitlesInfo.discNumber = atoi(GetChildElementText(pSubtitleElmt, "cd").c_str()); + pSubtitlesInfo.discCount = pSubtitlesInfo.discNumber; + pSubtitlesInfo.downloadCount = atoi(GetChildElementText(pSubtitleElmt, "downloads").c_str()); + + auto pSubtitleChildElmt = pSubtitleElmt->FirstChildElement("urls"); + if (pSubtitleChildElmt) { + auto pURLElement = pSubtitleChildElmt->FirstChildElement("url"); + while (pURLElement) { + if (pURLElement->Attribute("what", "download")) { + pSubtitlesInfo.url = pURLElement->GetText(); + } + if (pURLElement->Attribute("what", "direct")) { + pSubtitlesInfo.id = pURLElement->GetText(); + } + pURLElement = pURLElement->NextSiblingElement(); + } + } + + if ((pSubtitleChildElmt = pSubtitleElmt->FirstChildElement("TVShow")) != nullptr) { + pSubtitlesInfo.seasonNumber = atoi(GetChildElementText(pSubtitleChildElmt, "season").c_str()); + pSubtitlesInfo.episodeNumber = atoi(GetChildElementText(pSubtitleChildElmt, "episode").c_str()); + } + + pSubtitlesInfo.fileName = pSubtitlesInfo.title + " " + std::to_string(pSubtitlesInfo.year); + if (pSubtitlesInfo.seasonNumber > 0) { + pSubtitlesInfo.fileName += StringFormat(" S%02d", pSubtitlesInfo.seasonNumber); + } + if (pSubtitlesInfo.episodeNumber > 0) { + pSubtitlesInfo.fileName += StringFormat("%sE%02d", (pSubtitlesInfo.seasonNumber > 0) ? "" : " ", pSubtitlesInfo.episodeNumber); + } + + bool found = false; + for (const auto& str : pSubtitlesInfo.releaseNames) { + if (pFileInfo.fileName.find(str) != std::string::npos) { + pSubtitlesInfo.fileName += " " + str; + found = true; + break; + } + } + + if (!found && !pSubtitlesInfo.releaseNames.empty()) { + pSubtitlesInfo.fileName += " " + pSubtitlesInfo.releaseNames.front(); + } + pSubtitlesInfo.fileName += GUESSED_NAME_POSTFIX; + + Set(pSubtitlesInfo); + pSubtitleElmt = pSubtitleElmt->NextSiblingElement(); + } + } + } + } + return searchResult; +} + +SRESULT titlovi::Download(SubtitlesInfo& pSubtitlesInfo) +{ + LOG(LOG_INPUT, pSubtitlesInfo.id.c_str()); + return DownloadInternal(pSubtitlesInfo.id, "", pSubtitlesInfo.fileContents); +} + +const std::set<std::string>& titlovi::Languages() const +{ + static std::once_flag initialized; + static std::set<std::string> result; + + std::call_once(initialized, [this]() { + for (const auto& iter : titlovi_languages) { + if (strlen(iter.name)) { + result.emplace(iter.name); + } + } + }); + return result; +} + +/****************************************************************************** +** ysubs +******************************************************************************/ + +SRESULT ysubs::Search(const SubtitlesInfo& pFileInfo) +{ + SRESULT searchResult = SR_UNDEFINED; + using namespace rapidjson; + + if (pFileInfo.year && pFileInfo.seasonNumber == -1 && pFileInfo.episodeNumber == -1) { + std::string urlApi(StringFormat("https://yts.ag/api/v2/list_movies.json?query_term=%s", UrlEncode(pFileInfo.title.c_str()))); + LOG(LOG_INPUT, urlApi.c_str()); + + std::string data; + searchResult = DownloadInternal(urlApi, "", data); + + Document d; + if (d.ParseInsitu(&data[0]).HasParseError()) { + return SR_FAILED; + } + + auto root = d.FindMember("data"); + if (root != d.MemberEnd()) { + auto iter = root->value.FindMember("movies"); + if ((iter != root->value.MemberEnd()) && (iter->value.IsArray())) { + std::set<std::string> imdb_ids; + for (auto elem = iter->value.Begin(); elem != iter->value.End(); ++elem) { + std::string imdb = elem->FindMember("imdb_code")->value.GetString(); + if (imdb_ids.find(imdb) == imdb_ids.end()) { + imdb_ids.insert(imdb); + + std::string urlSubs(StringFormat("http://api.ysubs.com/subs/%s", imdb.c_str())); + LOG(LOG_INPUT, urlSubs.c_str()); + + std::string data1; + searchResult = DownloadInternal(urlSubs, "", data1); + Document d1; + if (d1.ParseInsitu(&data1[0]).HasParseError()) { + return SR_FAILED; + } + + auto iter1 = d1.FindMember("subs"); + if (iter1 != d1.MemberEnd()) { + iter1 = iter1->value.FindMember(imdb.c_str()); + if (iter1 != d1.MemberEnd()) { + for (auto elem1 = iter1->value.MemberBegin(); elem1 != iter1->value.MemberEnd(); ++elem1) { + std::string lang = elem1->name.GetString(); + std::string lang_code; + for (const auto& language : ysubs_languages) { + if (lang == language.name) { + lang_code = language.code; + } + } + if (CheckLanguage(lang_code)) { + for (auto elem2 = elem1->value.Begin(); elem2 != elem1->value.End(); ++elem2) { + SubtitlesInfo pSubtitlesInfo; + + pSubtitlesInfo.title = elem->FindMember("title")->value.GetString(); + pSubtitlesInfo.languageCode = lang_code; + pSubtitlesInfo.languageName = UTF16To8(ISOLang::ISO639XToLanguage(pSubtitlesInfo.languageCode.c_str())); + pSubtitlesInfo.releaseNames.emplace_back("YIFY"); + pSubtitlesInfo.imdbid = imdb; + pSubtitlesInfo.year = elem->FindMember("year")->value.GetInt(); + pSubtitlesInfo.discNumber = 1; + pSubtitlesInfo.discCount = 1; + + pSubtitlesInfo.url = "http://www.yifysubtitles.com/movie-imdb/" + imdb; + std::string str = elem2->FindMember("url")->value.GetString(); + pSubtitlesInfo.id = "http://www.yifysubtitles.com" + str; + pSubtitlesInfo.hearingImpaired = elem2->FindMember("hi")->value.GetInt(); + pSubtitlesInfo.corrected = elem2->FindMember("rating")->value.GetInt(); + + pSubtitlesInfo.fileName = pFileInfo.fileName; + pSubtitlesInfo.fileName += GUESSED_NAME_POSTFIX; + + Set(pSubtitlesInfo); + } + } + } + } + } + } + } + } + } + } + + return searchResult; +} + +SRESULT ysubs::Download(SubtitlesInfo& pSubtitlesInfo) +{ + LOG(LOG_INPUT, pSubtitlesInfo.id.c_str()); + return DownloadInternal(pSubtitlesInfo.id, "", pSubtitlesInfo.fileContents); +} + +const std::set<std::string>& ysubs::Languages() const +{ + static std::once_flag initialized; + static std::set<std::string> result; + + std::call_once(initialized, [this]() { + for (const auto& iter : ysubs_languages) { + if (strlen(iter.code)) { + result.emplace(iter.code); + } + } + }); + return result; +} + +/****************************************************************************** +** Napisy24 +******************************************************************************/ + +SRESULT Napisy24::Search(const SubtitlesInfo& pFileInfo) +{ + stringMap headers({ + { "User-Agent", UserAgent() }, + { "Content-Type", "application/x-www-form-urlencoded" } + }); + std::string data; + std::string url = "http://napisy24.pl/run/CheckSubAgent.php"; + std::string content = "postAction=CheckSub"; + content += "&ua=mpc-hc"; + content += "&ap=mpc-hc"; + content += "&fh=" + pFileInfo.fileHash; + content += "&fs=" + std::to_string(pFileInfo.fileSize); + content += "&fn=" + pFileInfo.fileName; + + LOG(LOG_INPUT, std::string(url + "?" + content).c_str()); + StringUpload(url, headers, content, data); + + if (data.length() < 4) { + return SR_FAILED; + } + + // Get status + std::string status = data.substr(0, 4); + if (status != "OK-2" && status != "OK-3") { + return SR_FAILED; + } + data.erase(0, 5); + + size_t infoEnd = data.find("||"); + if (infoEnd == std::string::npos) { + return SR_FAILED; + } + + // Search already returns whole file + SubtitlesInfo subtitleInfo; + subtitleInfo.fileContents = data.substr(infoEnd + 2); + subtitleInfo.languageCode = "pl"; // API doesn't support other languages yet. + + // Remove subtitle data + data.erase(infoEnd); + + std::unordered_map<std::string, std::string> subtitleInfoMap; + std::istringstream stringStream(data); + std::string entry; + while (std::getline(stringStream, entry, '|')) { + auto delimPos = entry.find(':'); + if (delimPos == std::string::npos) { + continue; + } + std::string key = entry.substr(0, delimPos); + if (entry.length() <= delimPos + 1) { + continue; + } + std::string value = entry.substr(delimPos + 1); + subtitleInfoMap[key] = value; + } + + subtitleInfo.url = "http://napisy24.pl/komentarze?napisId=" + subtitleInfoMap["napisId"]; + subtitleInfo.title = subtitleInfoMap["ftitle"]; + subtitleInfo.imdbid = subtitleInfoMap["fimdb"]; + + auto it = subtitleInfoMap.find("fyear"); + if (it != subtitleInfoMap.end()) { + subtitleInfo.year = std::stoi(it->second); + } + + it = subtitleInfoMap.find("fps"); + if (it != subtitleInfoMap.end()) { + subtitleInfo.frameRate = std::stod(it->second); + } + + int hour, minute, second; + if (sscanf_s(subtitleInfoMap["time"].c_str(), "%02d:%02d:%02d", &hour, &minute, &second) == 3) { + subtitleInfo.lengthMs = ((hour * 60 + minute) * 60 + second) * 1000; + } + + subtitleInfo.fileName = pFileInfo.fileName + "." + pFileInfo.fileExtension; + subtitleInfo.discNumber = 1; + subtitleInfo.discCount = 1; + + Set(subtitleInfo); + + return SR_SUCCEEDED; +} + +SRESULT Napisy24::Hash(SubtitlesInfo& pFileInfo) +{ + pFileInfo.fileHash = StringFormat("%016I64x", GenerateOSHash(pFileInfo)); + LOG(LOG_OUTPUT, pFileInfo.fileHash.c_str()); + return SR_SUCCEEDED; +} + +SRESULT Napisy24::Download(SubtitlesInfo& subtitlesInfo) +{ + LOG(LOG_INPUT, subtitlesInfo.url.c_str()); + return subtitlesInfo.fileContents.empty() ? SR_FAILED : SR_SUCCEEDED; +} + +const std::set<std::string>& Napisy24::Languages() const +{ + static std::set<std::string> result = {"pl"}; + return result; +} |