diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2015-05-24 19:05:24 +0300 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2015-05-24 19:05:24 +0300 |
commit | a693c83161775c5bb9ad4d88c56af68697ea296b (patch) | |
tree | 81bfdab037519d3798404974b77517eec1363a5c | |
parent | 582a845524774ecea69fed7232c3cd56fe1ed3a1 (diff) | |
parent | c82ee9a4e9fb5727b946bb0fe0f805f8ce8c044c (diff) |
Merge branch 'master' of github.com:moses-smt/mosesdecoder
-rw-r--r-- | Jamroot | 2 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/test-http-client.cc | 27 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/test-xml-escaping.cc | 13 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_http_client.cc | 200 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_http_client.h | 57 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_sampling_bias.cc | 36 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mmsapt.cpp | 10 |
7 files changed, 321 insertions, 24 deletions
@@ -89,7 +89,7 @@ if [ path.exists $(home)/moses-environment.jam ] include $(TOP)/jam-files/check-environment.jam ; # get resource locations # from environment variables include $(TOP)/jam-files/xmlrpc-c.jam ; # xmlrpc-c stuff for the server -include $(TOP)/jam-files/curlpp.jam ; # curlpp stuff for bias lookup (MMT only) +# include $(TOP)/jam-files/curlpp.jam ; # curlpp stuff for bias lookup (MMT only) # exit "done" : 0 ; diff --git a/moses/TranslationModel/UG/mm/test-http-client.cc b/moses/TranslationModel/UG/mm/test-http-client.cc new file mode 100644 index 000000000..f50f3b468 --- /dev/null +++ b/moses/TranslationModel/UG/mm/test-http-client.cc @@ -0,0 +1,27 @@ +// -*- c++ -*- +#include "ug_http_client.h" + +int main(int argc, char* argv[]) +{ + try + { + if (argc != 2) + { + std::cout << "Usage: async_client <url>\n"; + std::cout << "Example:\n"; + std::cout << " async_client www.boost.org/LICENSE_1_0.txt\n"; + return 1; + } + + boost::asio::io_service io_service; + Moses::http_client c(io_service, argv[1]); + io_service.run(); + std::cout << c.content() << std::endl; + } + catch (std::exception& e) + { + std::cout << "Exception: " << e.what() << "\n"; + } + + return 0; +} diff --git a/moses/TranslationModel/UG/mm/test-xml-escaping.cc b/moses/TranslationModel/UG/mm/test-xml-escaping.cc new file mode 100644 index 000000000..a99471a9b --- /dev/null +++ b/moses/TranslationModel/UG/mm/test-xml-escaping.cc @@ -0,0 +1,13 @@ +#include <iostream> +#include <string> +#include <iomanip> +#include "ug_http_client.h" + +using namespace std; +int main() +{ + string line; + while (getline(cin,line)) + cout << Moses::uri_encode(line) << endl; +} + diff --git a/moses/TranslationModel/UG/mm/ug_http_client.cc b/moses/TranslationModel/UG/mm/ug_http_client.cc new file mode 100644 index 000000000..1d6d70edb --- /dev/null +++ b/moses/TranslationModel/UG/mm/ug_http_client.cc @@ -0,0 +1,200 @@ +#include "ug_http_client.h" +namespace Moses +{ +using boost::asio::ip::tcp; + +std::string http_client::content() const { return m_content.str(); } + +http_client:: +http_client(boost::asio::io_service& io_service, + const std::string& server, const std::string& path) + : resolver_(io_service), socket_(io_service) +{ + init(server,path); +} + +http_client:: +http_client(boost::asio::io_service& io_service, std::string url) + : resolver_(io_service), socket_(io_service) +{ + size_t p = url.find("://"); + if (p < url.size()) url.erase(0,p+3); + p = url.find("/"); + if (p < url.size()) + init(url.substr(0,p),url.substr(p)); + else + init(url,"/"); +} + +void +http_client:: +init(std::string const& server, std::string const& path) +{ + // Form the request. We specify the "Connection: close" header so + // that the server will close the socket after transmitting the + // response. This will allow us to treat all data up until the EOF + // as the content. + + std::ostream request_stream(&request_); + request_stream << "GET " << path << " HTTP/1.0\r\n"; + request_stream << "Host: " << server << "\r\n"; + request_stream << "Accept: */*\r\n"; + request_stream << "Connection: close\r\n\r\n"; + + // Start an asynchronous resolve to translate the server and service names + // into a list of endpoints. + tcp::resolver::query query(server, "http"); + resolver_.async_resolve(query, + boost::bind(&http_client::handle_resolve, this, + boost::asio::placeholders::error, + boost::asio::placeholders::iterator)); + +} + +void +http_client:: +handle_resolve(const boost::system::error_code& err, + tcp::resolver::iterator endpoint_iterator) +{ + if (!err) + { + // Attempt a connection to the first endpoint in the list. Each endpoint + // will be tried until we successfully establish a connection. + tcp::endpoint endpoint = *endpoint_iterator; + socket_.async_connect(endpoint, + boost::bind(&http_client::handle_connect, this, + boost::asio::placeholders::error, ++endpoint_iterator)); + } + else + { + m_error << "Error: " << err.message() << "\n"; + } +} + +void +http_client:: +handle_connect(const boost::system::error_code& err, + tcp::resolver::iterator endpoint_iterator) +{ + if (!err) + { + // The connection was successful. Send the request. + boost::asio::async_write(socket_, request_, + boost::bind(&http_client::handle_write_request, this, + boost::asio::placeholders::error)); + } + else if (endpoint_iterator != tcp::resolver::iterator()) + { + // The connection failed. Try the next endpoint in the list. + socket_.close(); + tcp::endpoint endpoint = *endpoint_iterator; + socket_.async_connect(endpoint, + boost::bind(&http_client::handle_connect, this, + boost::asio::placeholders::error, ++endpoint_iterator)); + } + else m_error << "Error: " << err.message() << "\n"; +} + +void +http_client:: +handle_write_request(const boost::system::error_code& err) +{ + using namespace boost::asio; + if (err) { m_error << "Error: " << err.message() << "\n"; return; } + + // Read the response status line. The response_ streambuf will + // automatically grow to accommodate the entire line. The growth may be + // limited by passing a maximum size to the streambuf constructor. + async_read_until(socket_, response_, "\r\n", + boost::bind(&http_client::handle_read_status_line, + this, placeholders::error)); +} + +void +http_client:: +handle_read_status_line(const boost::system::error_code& err) +{ + if (err) { m_error << "Error: " << err << "\n"; return; } + + using namespace boost::asio; + // Check that response is OK. + std::istream response_stream(&response_); + response_stream >> m_http_version >> m_status_code; + std::getline(response_stream, m_status_message); + if (!response_stream || m_http_version.substr(0, 5) != "HTTP/") + m_error << "Invalid response\n"; + else if (m_status_code != 200) + m_error << "Response returned with status code " << m_status_code << "\n"; + else // Read the response headers, which are terminated by a blank line. + async_read_until(socket_, response_, "\r\n\r\n", + boost::bind(&http_client::handle_read_headers, this, + placeholders::error)); +} + + +void +http_client:: +handle_read_headers(const boost::system::error_code& err) +{ + if (err) { m_error << "Error: " << err << "\n"; return; } + + // Process the response headers. + std::istream response_stream(&response_); + std::string line; + while (std::getline(response_stream, line) && line != "\r") + m_header.push_back(line); + + // Write whatever content we already have to output. + if (response_.size() > 0) + m_content << &response_; + + using namespace boost::asio; + // Start reading remaining data until EOF. + async_read(socket_, response_, transfer_at_least(1), + boost::bind(&http_client::handle_read_content, this, + placeholders::error)); +} + +void +http_client:: +handle_read_content(const boost::system::error_code& err) +{ + using namespace boost::asio; + if(!err) + { + // Write all of the data that has been read so far. + // Then continue reading remaining data until EOF. + m_content << &response_; + async_read(socket_, response_, transfer_at_least(1), + boost::bind(&http_client::handle_read_content, this, + placeholders::error)); + } + else if (err != boost::asio::error::eof) + { + m_error << "Error: " << err << "\n"; + } +} + +std::string +uri_encode(std::string const& in) +{ + char buf[3 * in.size() + 1]; + size_t i = 0; + for (unsigned char const* c = (unsigned char const*)in.c_str(); *c; ++c) + { + // cout << *c << " " << int(*c) << endl; + if (*c == ' ') buf[i++] = '+'; + else if (*c == '.' || *c == '~' || *c == '_' || *c == '-') buf[i++] = *c; + else if (*c < '0') i += sprintf(buf+i, "%%%x", int(*c)); + else if (*c <= '9') buf[i++] = *c; + else if (*c < 'A') i += sprintf(buf+i, "%%%x", int(*c)); + else if (*c <= 'Z') buf[i++] = *c; + else if (*c < 'a') i += sprintf(buf+i, "%%%x", int(*c)); + else if (*c <= 'z') buf[i++] = *c; + else i += sprintf(buf+i, "%%%x", int(*c)); + } + buf[i] = 0; + return std::string(buf); +} + +} diff --git a/moses/TranslationModel/UG/mm/ug_http_client.h b/moses/TranslationModel/UG/mm/ug_http_client.h new file mode 100644 index 000000000..53ee258f9 --- /dev/null +++ b/moses/TranslationModel/UG/mm/ug_http_client.h @@ -0,0 +1,57 @@ +// -*- c++ -*- +// Adapted by Ulrich Germann from: +// async_client.cpp +// ~~~~~~~~~~~~~~~~ +// +// Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once +#include <iostream> +#include <istream> +#include <ostream> +#include <string> +#include <sstream> +#include <boost/asio.hpp> +#include <boost/bind.hpp> + +namespace Moses +{ +using boost::asio::ip::tcp; + +std::string uri_encode(std::string const& in); + +class http_client +{ + std::ostringstream m_content; + std::vector<std::string> m_header; + std::string m_http_version; + unsigned int m_status_code; + std::string m_status_message; + std::ostringstream m_error; + +public: + http_client(boost::asio::io_service& io_service, std::string url); + http_client(boost::asio::io_service& io_service, + const std::string& server, const std::string& path); +private: + void init(std::string const& server, std::string const& path); + void handle_resolve(const boost::system::error_code& err, + tcp::resolver::iterator endpoint_iterator); + void handle_connect(const boost::system::error_code& err, + tcp::resolver::iterator endpoint_iterator); + void handle_write_request(const boost::system::error_code& err); + void handle_read_status_line(const boost::system::error_code& err); + void handle_read_headers(const boost::system::error_code& err); + void handle_read_content(const boost::system::error_code& err); + tcp::resolver resolver_; + tcp::socket socket_; + boost::asio::streambuf request_; + boost::asio::streambuf response_; +public: + std::string content() const; +}; + +} diff --git a/moses/TranslationModel/UG/mm/ug_sampling_bias.cc b/moses/TranslationModel/UG/mm/ug_sampling_bias.cc index ebe1ce91d..da408dfb3 100644 --- a/moses/TranslationModel/UG/mm/ug_sampling_bias.cc +++ b/moses/TranslationModel/UG/mm/ug_sampling_bias.cc @@ -3,11 +3,15 @@ #include <boost/foreach.hpp> #include "moses/Timer.h" -#ifdef HAVE_CURLPP -#include <curlpp/Options.hpp> -#include <curlpp/cURLpp.hpp> -#include <curlpp/Easy.hpp> -#endif +// #ifdef HAVE_CURLPP +// #include <curlpp/Options.hpp> +// #include <curlpp/cURLpp.hpp> +// #include <curlpp/Easy.hpp> +// #endif + +// #ifdef WITH_MMT_BIAS_CLIENT +#include "ug_http_client.h" +// #endif namespace Moses { @@ -15,21 +19,17 @@ namespace Moses { using ugdiss::id_type; -#ifdef HAVE_CURLPP + // #ifdef WITH_MMT_BIAS_CLIENT std::string query_bias_server(std::string const& url, std::string const& text) { - // communicate with the bias server; resuts will be in ... - std::ostringstream os; - curlpp::Easy myRequest; - std::string query = url+curlpp::escape(text); - myRequest.setOpt(new curlpp::options::Url(query)); - curlpp::options::WriteStream ws(&os); - myRequest.setOpt(ws); // Give it to your request - myRequest.perform(); // This will output to os - return os.str(); + std::string query = url+uri_encode(text); + boost::asio::io_service io_service; + Moses::http_client c(io_service, query); + io_service.run(); + return c.content(); } -#endif + // #endif DocumentBias ::DocumentBias @@ -40,13 +40,13 @@ namespace Moses : m_sid2docid(sid2doc) , m_bias(docname2docid.size(), 0) { -#ifdef HAVE_CURLPP + // #ifdef HAVE_CURLPP Timer timer; if (log) timer.start(NULL); std::string json = query_bias_server(server_url, text); init_from_json(json, docname2docid, log); if (log) *log << "Bias query took " << timer << " seconds." << std::endl; -#endif + // #endif } void diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index 4ce775877..f05c0d59b 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -1,8 +1,8 @@ -#ifdef HAVE_CURLPP -#include <curlpp/Options.hpp> -#include <curlpp/cURLpp.hpp> -#include <curlpp/Easy.hpp> -#endif +// #ifdef HAVE_CURLPP +// #include <curlpp/Options.hpp> +// #include <curlpp/cURLpp.hpp> +// #include <curlpp/Easy.hpp> +// #endif #include "mmsapt.h" #include <boost/foreach.hpp> |