Fix Lua C function stack overflow during encoding

Ensure there are enough Lua stack slots available before descending into
another table during encoding. This fixes a segfault when encoding
deeply nested tables.

This bug wasn't noticed earlier due to the previous limit of 20 nested
tables.

2 files changed, 21 insertions, 7 deletions

diff --git a/lua_cjson.c b/lua_cjson.c
index 00fa2dd..4a2ce0f 100644
--- a/lua_cjson.c
+++ b/lua_cjson.c
@@ -564,12 +564,17 @@ static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json)
 static void json_check_encode_depth(lua_State *l, json_config_t *cfg,
                                     int current_depth, strbuf_t *json)
 {
-    if (current_depth > cfg->encode_max_depth) {
-        if (!cfg->encode_keep_buffer)
-            strbuf_free(json);
-        luaL_error(l, "Cannot serialise, excessive nesting (%d)",
-                   current_depth);
-    }
+    /* Ensure there are enough slots free to traverse a table (key, value).
+     * luaL_error() and other Lua API functions use space from the
+     * "EXTRA_STACK" reserve. */
+    if (current_depth <= cfg->encode_max_depth && lua_checkstack(l, 2))
+        return;
+
+    if (!cfg->encode_keep_buffer)
+        strbuf_free(json);
+
+    luaL_error(l, "Cannot serialise, excessive nesting (%d)",
+               current_depth);
 }
 
 static void json_append_data(lua_State *l, json_config_t *cfg,
@@ -692,9 +697,9 @@ static void json_append_data(lua_State *l, json_config_t *cfg,
             strbuf_append_mem(json, "false", 5);
         break;
     case LUA_TTABLE:
-        len = lua_array_length(l, cfg, json);
         current_depth++;
         json_check_encode_depth(l, cfg, current_depth, json);
+        len = lua_array_length(l, cfg, json);
         if (len > 0)
             json_append_array(l, cfg, current_depth, json, len);
         else
diff --git a/tests/test.lua b/tests/test.lua
index a827e6a..4c00453 100755
--- a/tests/test.lua
+++ b/tests/test.lua
@@ -67,6 +67,12 @@ function load_testdata()
     data.table_cycle = {}
     data.table_cycle[1] = data.table_cycle
 
+    local big = {}
+    for i = 1, 1100 do
+        big = { { 10, false, true, cjson.null }, "string", a = big }
+    end
+    data.deeply_nested_data = big
+
     return data
 end
 
@@ -185,6 +191,9 @@ local cjson_tests = {
       false, { "Cannot serialise, excessive nesting (6)" } },
     { "Set encode_max_depth(1000)",
       json.encode_max_depth, { 1000 }, true, { 1000 } },
+    { "Encode deeply nested data [throw error]",
+      json.encode, { testdata.deeply_nested_data },
+      false, { "Cannot serialise, excessive nesting (1001)" } },
 
     -- Test encoding simple types
     { "Encode null",