Merge branch 'master' into xml-ng

Conflicts:
	src/adhoc.erl
	src/cyrsasl_oauth.erl
	src/ejabberd_c2s.erl
	src/ejabberd_config.erl
	src/ejabberd_service.erl
	src/gen_mod.erl
	src/mod_admin_extra.erl
	src/mod_announce.erl
	src/mod_carboncopy.erl
	src/mod_client_state.erl
	src/mod_configure.erl
	src/mod_echo.erl
	src/mod_mam.erl
	src/mod_muc.erl
	src/mod_muc_room.erl
	src/mod_offline.erl
	src/mod_pubsub.erl
	src/mod_stats.erl
	src/node_flat_sql.erl
	src/randoms.erl

1 files changed, 186 insertions, 133 deletions

diff --git a/src/mod_http_api.erl b/src/mod_http_api.erl
index 1578be964..881587ede 100644
--- a/src/mod_http_api.erl
+++ b/src/mod_http_api.erl
@@ -101,7 +101,7 @@
 
 -define(AC_ALLOW_HEADERS,
         {<<"Access-Control-Allow-Headers">>,
-         <<"Content-Type">>}).
+         <<"Content-Type, Authorization, X-Admin">>}).
 
 -define(AC_MAX_AGE,
         {<<"Access-Control-Max-Age">>, <<"86400">>}).
@@ -118,9 +118,11 @@
 %% -------------------
 
 start(_Host, _Opts) ->
+    ejabberd_access_permissions:register_permission_addon(?MODULE, fun permission_addon/0),
     ok.
 
 stop(_Host) ->
+    ejabberd_access_permissions:unregister_permission_addon(?MODULE),
     ok.
 
 depends(_Host, _Opts) ->
@@ -130,79 +132,39 @@ depends(_Host, _Opts) ->
 %% basic auth
 %% ----------
 
-check_permissions(Request, Command) ->
-    case catch binary_to_existing_atom(Command, utf8) of
-        Call when is_atom(Call) ->
-            {ok, CommandPolicy} = ejabberd_commands:get_command_policy(Call),
-            check_permissions2(Request, Call, CommandPolicy);
-        _ ->
-            unauthorized_response()
-    end.
-
-check_permissions2(#request{auth = HTTPAuth, headers = Headers}, Call, _)
-  when HTTPAuth /= undefined ->
-    Admin =
-        case lists:keysearch(<<"X-Admin">>, 1, Headers) of
-            {value, {_, <<"true">>}} -> true;
-            _ -> false
-        end,
-    Auth =
-        case HTTPAuth of
+extract_auth(#request{auth = HTTPAuth, ip = {IP, _}}) ->
+    Info = case HTTPAuth of
             {SJID, Pass} ->
                 case jid:from_string(SJID) of
-                    #jid{user = User, server = Server} ->
+		       #jid{luser = User, lserver = Server} ->
                         case ejabberd_auth:check_password(User, <<"">>, Server, Pass) of
-                            true -> {ok, {User, Server, Pass, Admin}};
-                            false -> false
+			       true ->
+				   #{usr => {User, Server, <<"">>}, caller_server => Server};
+			       false ->
+				   {error, invalid_auth}
                         end;
                     _ ->
-                        false
+			   {error, invalid_auth}
                 end;
             {oauth, Token, _} ->
-                case oauth_check_token(Call, Token) of
-                    {ok, user, {User, Server}} ->
-                        {ok, {User, Server, {oauth, Token}, Admin}};
-                    {ok, server_admin} ->  %% token whas generated using issue_token command line
-                        {ok, admin};
-                    false ->
-                        false
+		   case ejabberd_oauth:check_token(Token) of
+		       {ok, {U, S}, Scope} ->
+			   #{usr => {U, S, <<"">>}, oauth_scope => Scope, caller_server => S};
+		       {false, Reason} ->
+			   {error, Reason}
                 end;
             _ ->
-                false
+		   #{}
         end,
-    case Auth of
-        {ok, A} -> {allowed, Call, A};
-        _ -> unauthorized_response()
-    end;
-check_permissions2(_Request, Call, open) ->
-    {allowed, Call, noauth};
-check_permissions2(#request{ip={IP, _Port}}, Call, _Policy) ->
-    Access = gen_mod:get_module_opt(global, ?MODULE, admin_ip_access,
-                                    fun(V) -> V end,
-                                    none),
-    Res = acl:match_rule(global, Access, IP),
-    case Res of
-        all ->
-            {allowed, Call, admin};
-        [all] ->
-            {allowed, Call, admin};
-        allow ->
-            {allowed, Call, admin};
-        Commands when is_list(Commands) ->
-            case lists:member(Call, Commands) of
-                true -> {allowed, Call, admin};
-                _ -> unauthorized_response()
-            end;
-        _E ->
-            {allowed, Call, noauth}
+    case Info of
+	Map when is_map(Map) ->
+	    Map#{caller_module => ?MODULE, ip => IP};
+	_ ->
+	    ?DEBUG("Invalid auth data: ~p", [Info]),
+	    Info
     end;
-check_permissions2(_Request, _Call, _Policy) ->
-    unauthorized_response().
-
-oauth_check_token(Scope, Token) when is_atom(Scope) ->
-    oauth_check_token(atom_to_binary(Scope, utf8), Token);
-oauth_check_token(Scope, Token) ->
-    ejabberd_oauth:check_token(Scope, Token).
+extract_auth(#request{ip = IP}) ->
+    #{ip => IP, caller_module => ?MODULE}.
 
 %% ------------------
 %% command processing
@@ -213,31 +175,24 @@ oauth_check_token(Scope, Token) ->
 process(_, #request{method = 'POST', data = <<>>}) ->
     ?DEBUG("Bad Request: no data", []),
     badrequest_response(<<"Missing POST data">>);
-process([Call], #request{method = 'POST', data = Data, ip = {IP, _} = IPPort} = Req) ->
+process([Call], #request{method = 'POST', data = Data, ip = IPPort} = Req) ->
     Version = get_api_version(Req),
     try
-        Args = case jiffy:decode(Data) of
-            List when is_list(List) -> List;
-            {List} when is_list(List) -> List;
-            Other -> [Other]
-        end,
+        Args = extract_args(Data),
         log(Call, Args, IPPort),
-        case check_permissions(Req, Call) of
-            {allowed, Cmd, Auth} ->
-                {Code, Result} = handle(Cmd, Auth, Args, Version, IP),
-                json_response(Code, jiffy:encode(Result));
-            %% Warning: check_permission direcly formats 401 reply if not authorized
-            ErrorResponse ->
-                ErrorResponse
-        end
-    catch _:{error,{_,invalid_json}} = _Err ->
+	perform_call(Call, Args, Req, Version)
+    catch
+        %% TODO We need to refactor to remove redundant error return formatting
+        throw:{error, unknown_command} ->
+            json_format({404, 44, <<"Command not found.">>});
+        _:{error,{_,invalid_json}} = _Err ->
 	    ?DEBUG("Bad Request: ~p", [_Err]),
 	    badrequest_response(<<"Invalid JSON input">>);
 	  _:_Error ->
             ?DEBUG("Bad Request: ~p ~p", [_Error, erlang:get_stacktrace()]),
             badrequest_response()
     end;
-process([Call], #request{method = 'GET', q = Data, ip = IP} = Req) ->
+process([Call], #request{method = 'GET', q = Data, ip = {IP, _}} = Req) ->
     Version = get_api_version(Req),
     try
         Args = case Data of
@@ -245,23 +200,48 @@ process([Call], #request{method = 'GET', q = Data, ip = IP} = Req) ->
                    _ -> Data
                end,
         log(Call, Args, IP),
-        case check_permissions(Req, Call) of
-            {allowed, Cmd, Auth} ->
-                {Code, Result} = handle(Cmd, Auth, Args, Version, IP),
-                json_response(Code, jiffy:encode(Result));
-            %% Warning: check_permission direcly formats 401 reply if not authorized
-            ErrorResponse ->
-                ErrorResponse
-        end
-    catch _:_Error ->
+	perform_call(Call, Args, Req, Version)
+    catch
+        %% TODO We need to refactor to remove redundant error return formatting
+        throw:{error, unknown_command} ->
+            json_format({404, 44, <<"Command not found.">>});
+        _:_Error ->
+
         ?DEBUG("Bad Request: ~p ~p", [_Error, erlang:get_stacktrace()]),
         badrequest_response()
     end;
-process([], #request{method = 'OPTIONS', data = <<>>}) ->
+process([_Call], #request{method = 'OPTIONS', data = <<>>}) ->
     {200, ?OPTIONS_HEADER, []};
+process(_, #request{method = 'OPTIONS'}) ->
+    {400, ?OPTIONS_HEADER, []};
 process(_Path, Request) ->
     ?DEBUG("Bad Request: no handler ~p", [Request]),
-    badrequest_response().
+    json_error(400, 40, <<"Missing command name.">>).
+
+perform_call(Command, Args, Req, Version) ->
+    case catch binary_to_existing_atom(Command, utf8) of
+	Call when is_atom(Call) ->
+	    case extract_auth(Req) of
+		{error, expired} -> invalid_token_response();
+		{error, not_found} -> invalid_token_response();
+		{error, invalid_auth} -> unauthorized_response();
+		{error, _} -> unauthorized_response();
+		Auth when is_map(Auth) ->
+		    Result = handle(Call, Auth, Args, Version),
+		    json_format(Result)
+	    end;
+	_ ->
+	    json_error(404, 40, <<"Endpoint not found.">>)
+    end.
+
+%% Be tolerant to make API more easily usable from command-line pipe.
+extract_args(<<"\n">>) -> [];
+extract_args(Data) ->
+    case jiffy:decode(Data) of
+        List when is_list(List) -> List;
+        {List} when is_list(List) -> List;
+        Other -> [Other]
+    end.
 
 % get API version N from last "vN" element in URL path
 get_api_version(#request{path = Path}) ->
@@ -282,8 +262,10 @@ get_api_version([]) ->
 %% command handlers
 %% ----------------
 
+%% TODO Check accept types of request before decided format of reply.
+
 % generic ejabberd command handler
-handle(Call, Auth, Args, Version, IP) when is_atom(Call), is_list(Args) ->
+handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
     case ejabberd_commands:get_command_format(Call, Auth, Version) of
         {ArgsSpec, _} when is_list(ArgsSpec) ->
             Args2 = [{jlib:binary_to_atom(Key), Value} || {Key, Value} <- Args],
@@ -300,7 +282,7 @@ handle(Call, Auth, Args, Version, IP) when is_atom(Call), is_list(Args) ->
                             [{Key, undefined}|Acc]
                     end, [], ArgsSpec),
 	    try
-		handle2(Call, Auth, match(Args2, Spec), Version, IP)
+          handle2(Call, Auth, match(Args2, Spec), Version)
 	    catch throw:not_found ->
 		    {404, <<"not_found">>};
 		  throw:{not_found, Why} when is_atom(Why) ->
@@ -314,7 +296,9 @@ handle(Call, Auth, Args, Version, IP) when is_atom(Call), is_list(Args) ->
 		  throw:{not_allowed, Msg} ->
 		    {401, iolist_to_binary(Msg)};
                   throw:{error, account_unprivileged} ->
-                    {401, iolist_to_binary(<<"Unauthorized: Account Unpriviledged">>)};
+        {403, 31, <<"Command need to be run with admin priviledge.">>};
+      throw:{error, access_rules_unauthorized} ->
+        {403, 32, <<"AccessRules: Account associated to token does not have the right to perform the operation.">>};
 		  throw:{invalid_parameter, Msg} ->
 		    {400, iolist_to_binary(Msg)};
 		  throw:{error, Why} when is_atom(Why) ->
@@ -337,10 +321,15 @@ handle(Call, Auth, Args, Version, IP) when is_atom(Call), is_list(Args) ->
             {400, <<"Error">>}
     end.
 
-handle2(Call, Auth, Args, Version, IP) when is_atom(Call), is_list(Args) ->
+handle2(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
     {ArgsF, _ResultF} = ejabberd_commands:get_command_format(Call, Auth, Version),
     ArgsFormatted = format_args(Args, ArgsF),
-    ejabberd_command(Auth, Call, ArgsFormatted, Version, IP).
+    case ejabberd_commands:execute_command2(Call, ArgsFormatted, Auth, Version) of
+	{error, Error} ->
+	    throw(Error);
+	Res ->
+	    format_command_result(Call, Auth, Res, Version)
+    end.
 
 get_elem_delete(A, L) ->
     case proplists:get_all_values(A, L) of
@@ -370,28 +359,47 @@ format_args(Args, ArgsFormat) ->
       L when is_list(L) -> exit({additional_unused_args, L})
     end.
 
-format_arg({array, Elements},
-	   {list, {ElementDefName, ElementDefFormat}})
+format_arg({Elements},
+	   {list, {_ElementDefName, {tuple, [{_Tuple1N, Tuple1S}, {_Tuple2N, Tuple2S}]} = Tuple}})
+    when is_list(Elements) andalso
+	 (Tuple1S == binary orelse Tuple1S == string) ->
+    lists:map(fun({F1, F2}) ->
+		      {format_arg(F1, Tuple1S), format_arg(F2, Tuple2S)};
+		 ({Val}) when is_list(Val) ->
+		      format_arg({Val}, Tuple)
+	      end, Elements);
+format_arg(Elements,
+	   {list, {_ElementDefName, {list, _} = ElementDefFormat}})
     when is_list(Elements) ->
-    lists:map(fun ({struct, [{ElementName, ElementValue}]}) when
-                        ElementDefName == ElementName ->
-		      format_arg(ElementValue, ElementDefFormat)
-	      end,
-	      Elements);
-format_arg({array, [{struct, Elements}]},
-	   {list, {ElementDefName, ElementDefFormat}})
+    [{format_arg(Element, ElementDefFormat)}
+     || Element <- Elements];
+format_arg(Elements,
+	   {list, {_ElementDefName, ElementDefFormat}})
     when is_list(Elements) ->
-    lists:map(fun ({ElementName, ElementValue}) ->
-		      true = ElementDefName == ElementName,
-		      format_arg(ElementValue, ElementDefFormat)
-	      end,
-	      Elements);
-format_arg({array, [{struct, Elements}]},
+    [format_arg(Element, ElementDefFormat)
+     || Element <- Elements];
+format_arg({[{Name, Value}]},
+	   {tuple, [{_Tuple1N, Tuple1S}, {_Tuple2N, Tuple2S}]})
+  when Tuple1S == binary;
+       Tuple1S == string ->
+    {format_arg(Name, Tuple1S), format_arg(Value, Tuple2S)};
+format_arg({Elements},
 	   {tuple, ElementsDef})
     when is_list(Elements) ->
-    FormattedList = format_args(Elements, ElementsDef),
-    list_to_tuple(FormattedList);
-format_arg({array, Elements}, {list, ElementsDef})
+    F = lists:map(fun({TElName, TElDef}) ->
+			  case lists:keyfind(atom_to_binary(TElName, latin1), 1, Elements) of
+			      {_, Value} ->
+				  format_arg(Value, TElDef);
+			      _ when TElDef == binary; TElDef == string ->
+				  <<"">>;
+			      _ ->
+				  ?ERROR_MSG("missing field ~p in tuple ~p", [TElName, Elements]),
+				  throw({invalid_parameter,
+					 io_lib:format("Missing field ~w in tuple ~w", [TElName, Elements])})
+			  end
+		  end, ElementsDef),
+    list_to_tuple(F);
+format_arg(Elements, {list, ElementsDef})
     when is_list(Elements) and is_atom(ElementsDef) ->
     [format_arg(Element, ElementsDef)
      || Element <- Elements];
@@ -405,7 +413,7 @@ format_arg(undefined, string) -> <<>>;
 format_arg(Arg, Format) ->
     ?ERROR_MSG("don't know how to format Arg ~p for format ~p", [Arg, Format]),
     throw({invalid_parameter,
-	   io_lib:format("Arg ~p is not in format ~p",
+	   io_lib:format("Arg ~w is not in format ~w",
 			 [Arg, Format])}).
 
 process_unicode_codepoints(Str) ->
@@ -420,18 +428,6 @@ process_unicode_codepoints(Str) ->
 match(Args, Spec) ->
     [{Key, proplists:get_value(Key, Args, Default)} || {Key, Default} <- Spec].
 
-ejabberd_command(Auth, Cmd, Args, Version, IP) ->
-    Access = case Auth of
-                 admin -> [];
-                 _ -> undefined
-             end,
-    case ejabberd_commands:execute_command(Access, Auth, Cmd, Args, Version, #{ip => IP}) of
-        {error, Error} ->
-            throw(Error);
-        Res ->
-            format_command_result(Cmd, Auth, Res, Version)
-    end.
-
 format_command_result(Cmd, Auth, Result, Version) ->
     {_, ResultFormat} = ejabberd_commands:get_command_format(Cmd, Auth, Version),
     case {ResultFormat, Result} of
@@ -439,10 +435,12 @@ format_command_result(Cmd, Auth, Result, Version) ->
 	    {200, 0};
 	{{_, rescode}, _} ->
 	    {200, 1};
-	{{_, restuple}, {V1, Text1}} when V1 == true; V1 == ok ->
-	    {200, iolist_to_binary(Text1)};
-	{{_, restuple}, {_, Text2}} ->
-	    {500, iolist_to_binary(Text2)};
+        {_, {error, ErrorAtom, Code, Msg}} ->
+            format_error_result(ErrorAtom, Code, Msg);
+        {{_, restuple}, {V, Text}} when V == true; V == ok ->
+            {200, iolist_to_binary(Text)};
+        {{_, restuple}, {ErrorAtom, Msg}} ->
+            format_error_result(ErrorAtom, 0, Msg);
 	{{_, {list, _}}, _V} ->
 	    {_, L} = format_result(Result, ResultFormat),
 	    {200, L};
@@ -470,6 +468,11 @@ format_result({Code, Text}, {Name, restuple}) ->
      {[{<<"res">>, Code == true orelse Code == ok},
        {<<"text">>, iolist_to_binary(Text)}]}};
 
+format_result(Code, {Name, restuple}) ->
+    {jlib:atom_to_binary(Name),
+     {[{<<"res">>, Code == true orelse Code == ok},
+       {<<"text">>, <<"">>}]}};
+
 format_result(Els, {Name, {list, {_, {tuple, [{_, atom}, _]}} = Fmt}}) ->
     {jlib:atom_to_binary(Name), {[format_result(El, Fmt) || El <- Els]}};
 
@@ -488,24 +491,74 @@ format_result(Tuple, {Name, {tuple, Def}}) ->
 format_result(404, {_Name, _}) ->
     "not_found".
 
+
+format_error_result(conflict, Code, Msg) ->
+    {409, Code, iolist_to_binary(Msg)};
+format_error_result(_ErrorAtom, Code, Msg) ->
+    {500, Code, iolist_to_binary(Msg)}.
+
 unauthorized_response() ->
-    unauthorized_response(<<"401 Unauthorized">>).
-unauthorized_response(Body) ->
-    json_response(401, jiffy:encode(Body)).
+    json_error(401, 10, <<"You are not authorized to call this command.">>).
+
+invalid_token_response() ->
+    json_error(401, 10, <<"Oauth Token is invalid or expired.">>).
+
+outofscope_response() ->
+    json_error(401, 11, <<"Token does not grant usage to command required scope.">>).
 
 badrequest_response() ->
     badrequest_response(<<"400 Bad Request">>).
 badrequest_response(Body) ->
     json_response(400, jiffy:encode(Body)).
 
+json_format({Code, Result}) ->
+    json_response(Code, jiffy:encode(Result));
+json_format({HTMLCode, JSONErrorCode, Message}) ->
+    json_error(HTMLCode, JSONErrorCode, Message).
+
 json_response(Code, Body) when is_integer(Code) ->
     {Code, ?HEADER(?CT_JSON), Body}.
 
+%% HTTPCode, JSONCode = integers
+%% message is binary
+json_error(HTTPCode, JSONCode, Message) ->
+    {HTTPCode, ?HEADER(?CT_JSON),
+     jiffy:encode({[{<<"status">>, <<"error">>},
+                    {<<"code">>, JSONCode},
+                    {<<"message">>, Message}]})
+    }.
+
 log(Call, Args, {Addr, Port}) ->
     AddrS = jlib:ip_to_list({Addr, Port}),
     ?INFO_MSG("API call ~s ~p from ~s:~p", [Call, Args, AddrS, Port]);
 log(Call, Args, IP) ->
     ?INFO_MSG("API call ~s ~p (~p)", [Call, Args, IP]).
 
+permission_addon() ->
+    Access = gen_mod:get_module_opt(global, ?MODULE, admin_ip_access,
+				    fun(V) -> V end,
+				    none),
+    Rules = acl:resolve_access(Access, global),
+    R = lists:filtermap(
+	fun({V, AclRules}) when V == all; V == [all]; V == [allow]; V == allow ->
+	       {true, {[{allow, AclRules}], {[<<"*">>], []}}};
+	   ({List, AclRules}) when is_list(List) ->
+	       {true, {[{allow, AclRules}], {List, []}}};
+	   (_) ->
+	       false
+	end, Rules),
+    case R of
+	[] ->
+	    none;
+	_ ->
+	    {_, Res} = lists:foldl(
+		fun({R2, L2}, {Idx, Acc}) ->
+		    {Idx+1, [{<<"'mod_http_api admin_ip_access' option compatibility shim ",
+				(integer_to_binary(Idx))/binary>>,
+			      {[?MODULE], [{access, R2}], L2}} | Acc]}
+		end, {1, []}, R),
+	    Res
+    end.
+
 mod_opt_type(admin_ip_access) -> fun acl:access_rules_validator/1;
 mod_opt_type(_) -> [admin_ip_access].