GPUShader: Add GLSL source modification pass to support enums

This uses a light parser / string modification pass to convert
C++ enum declaration syntax to GLSL compatible one.

GLSL having no support for enums, we are forced to convert the
enum values to a series of constant uints.

The parser (not really one by the way), being stupidly simple,
will not change anything to the values and thus make some C++
syntax (like omitting the values) not work.

The string replacement happens on all GLSL files on startup.
I did not measure significant changes in blender startup speed.
There is plans to do all of this at compile time.

We limit the scope of the search to `.h` and `.hh` files to prevent
confusing syntax in `.glsl` files.

There is basic error reporting with file, line and char logging
for easy debuggabiliy.

The requirements to use this enum sharing system are already listed in
`gpu_shader_shared_utils.h` and repeated on top of the preprocessor
function.

4 files changed, 201 insertions, 11 deletions

diff --git a/source/blender/draw/CMakeLists.txt b/source/blender/draw/CMakeLists.txt
index 94a1a0dfeba..0c5b158ac80 100644
--- a/source/blender/draw/CMakeLists.txt
+++ b/source/blender/draw/CMakeLists.txt
@@ -531,7 +531,7 @@ set(GLSL_SOURCE_CONTENT "")
 foreach(GLSL_FILE ${GLSL_SRC})
   get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
   string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
-  string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\"\)\n")
+  string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
 endforeach()
 
 set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_draw_source_list.h")
diff --git a/source/blender/gpu/CMakeLists.txt b/source/blender/gpu/CMakeLists.txt
index c898392dc7e..49b6b478148 100644
--- a/source/blender/gpu/CMakeLists.txt
+++ b/source/blender/gpu/CMakeLists.txt
@@ -393,7 +393,7 @@ set(GLSL_SOURCE_CONTENT "")
 foreach(GLSL_FILE ${GLSL_SRC})
   get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
   string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
-  string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\"\)\n")
+  string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
 endforeach()
 
 set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_gpu_source_list.h")
diff --git a/source/blender/gpu/intern/gpu_shader_dependency.cc b/source/blender/gpu/intern/gpu_shader_dependency.cc
index 2d56e1cec05..2333590810f 100644
--- a/source/blender/gpu/intern/gpu_shader_dependency.cc
+++ b/source/blender/gpu/intern/gpu_shader_dependency.cc
@@ -34,7 +34,7 @@
 #include "gpu_shader_dependency_private.h"
 
 extern "C" {
-#define SHADER_SOURCE(datatoc, filename) extern char datatoc[];
+#define SHADER_SOURCE(datatoc, filename, filepath) extern char datatoc[];
 #include "glsl_draw_source_list.h"
 #include "glsl_gpu_source_list.h"
 #undef SHADER_SOURCE
@@ -45,13 +45,16 @@ namespace blender::gpu {
 using GPUSourceDictionnary = Map<StringRef, struct GPUSource *>;
 
 struct GPUSource {
+  StringRefNull fullpath;
   StringRefNull filename;
   StringRefNull source;
   Vector<GPUSource *> dependencies;
   bool dependencies_init = false;
   shader::BuiltinBits builtins = (shader::BuiltinBits)0;
+  std::string processed_source;
 
-  GPUSource(const char *file, const char *datatoc) : filename(file), source(datatoc)
+  GPUSource(const char *path, const char *file, const char *datatoc)
+      : fullpath(path), filename(file), source(datatoc)
   {
     /* Scan for builtins. */
     /* FIXME: This can trigger false positive caused by disabled #if blocks. */
@@ -99,6 +102,194 @@ struct GPUSource {
     if (source.find("gl_WorkGroupSize", 0)) {
       builtins |= shader::BuiltinBits::WORK_GROUP_SIZE;
     }
+
+    /* TODO(fclem): We could do that at compile time. */
+    /* Limit to shared header files to avoid the temptation to use C++ syntax in .glsl files. */
+    if (filename.endswith(".h") || filename.endswith(".hh")) {
+      enum_preprocess();
+    }
+  };
+
+  bool is_in_comment(const StringRef &input, int64_t offset)
+  {
+    return (input.rfind("/*", offset) > input.rfind("*/", offset)) ||
+           (input.rfind("//", offset) > input.rfind("\n", offset));
+  }
+
+  template<bool check_whole_word = true, bool reversed = false, typename T>
+  int64_t find_str(const StringRef &input, const T keyword, int64_t offset = 0)
+  {
+    while (1) {
+      if constexpr (reversed) {
+        offset = input.rfind(keyword, offset);
+      }
+      else {
+        offset = input.find(keyword, offset);
+      }
+      if (offset > 0) {
+        if constexpr (check_whole_word) {
+          /* Fix false positive if something has "enum" as suffix. */
+          char previous_char = input[offset - 1];
+          if (!(ELEM(previous_char, '\n', '\t', ' ', ':'))) {
+            offset += (reversed) ? -1 : 1;
+            continue;
+          }
+        }
+        /* Fix case where the keyword is in a comment. */
+        if (is_in_comment(input, offset)) {
+          offset += (reversed) ? -1 : 1;
+          continue;
+        }
+      }
+      return offset;
+    }
+  }
+
+  void print_error(const StringRef &input, int64_t offset, const StringRef message)
+  {
+    std::cout << " error: " << message << "\n";
+    StringRef sub = input.substr(0, offset);
+    int64_t line_number = std::count(sub.begin(), sub.end(), '\n') + 1;
+    int64_t line_end = input.find("\n", offset);
+    int64_t line_start = input.rfind("\n", offset) + 1;
+    int64_t char_number = offset - line_start + 1;
+    char line_prefix[16] = "";
+    SNPRINTF(line_prefix, "%5ld | ", line_number);
+
+    /* TODO Use clog. */
+
+    std::cout << fullpath << ":" << line_number << ":" << char_number;
+
+    std::cout << " error: " << message << "\n";
+    std::cout << line_prefix << input.substr(line_start, line_end - line_start) << "\n";
+    std::cout << "      | ";
+    for (int64_t i = 0; i < char_number - 1; i++) {
+      std::cout << " ";
+    }
+    std::cout << "^\n";
+  }
+
+  /**
+   * Transform C,C++ enum declaration into GLSL compatible defines and constants:
+   *
+   * \code{.cpp}
+   * enum eMyEnum : uint32_t {
+   *   ENUM_1 = 0u,
+   *   ENUM_2 = 1u,
+   *   ENUM_3 = 2u,
+   * };
+   * \endcode
+   *
+   * or
+   *
+   * \code{.c}
+   * enum eMyEnum {
+   *   ENUM_1 = 0u,
+   *   ENUM_2 = 1u,
+   *   ENUM_3 = 2u,
+   * };
+   * \endcode
+   *
+   * becomes
+   *
+   * \code{.glsl}
+   * #define eMyEnum uint
+   * const uint ENUM_1 = 0u, ENUM_2 = 1u, ENUM_3 = 2u;
+   * \endcode
+   *
+   * IMPORTANT: This has some requirements:
+   * - Enums needs to have underlying types specified to uint32_t to make them usable in UBO/SSBO.
+   * - All values needs to be specified using constant literals to avoid compiler differencies.
+   * - All values needs to have the 'u' suffix to avoid GLSL compiler errors.
+   */
+  void enum_preprocess(void)
+  {
+    const StringRefNull input = source;
+    std::string output = "";
+    int64_t cursor = 0;
+    int64_t last_pos = 0;
+    const bool is_cpp = filename.endswith(".hh");
+
+#define find_keyword find_str<true, false>
+#define find_token find_str<false, false>
+#define rfind_token find_str<false, true>
+#define CHECK(test_value, str, ofs, msg) \
+  if ((test_value) == -1) { \
+    print_error(str, ofs, msg); \
+    cursor++; \
+    continue; \
+  }
+
+    while (1) {
+      cursor = find_keyword(input, "enum ", cursor);
+      if (cursor == -1) {
+        break;
+      }
+      /* Output anything between 2 enums blocks. */
+      output += input.substr(last_pos, cursor - last_pos);
+
+      /* Extract enum type name. */
+      int64_t name_start = input.find(" ", cursor);
+
+      int64_t values_start = find_token(input, '{', cursor);
+      CHECK(values_start, input, cursor, "Malformed enum class. Expected \'{\' after typename.");
+
+      StringRef enum_name = input.substr(name_start, values_start - name_start);
+      if (is_cpp) {
+        int64_t name_end = find_token(enum_name, ":");
+        CHECK(name_end, input, name_start, "Expected \':\' after C++ enum name.");
+
+        int64_t underlying_type = find_keyword(enum_name, "uint32_t", name_end);
+        CHECK(underlying_type, input, name_start, "C++ enums needs uint32_t underlying type.");
+
+        enum_name = input.substr(name_start, name_end);
+      }
+
+      output += "#define " + enum_name + " uint\n";
+
+      /* Extract enum values. */
+      int64_t values_end = find_token(input, '}', values_start);
+      CHECK(values_end, input, cursor, "Malformed enum class. Expected \'}\' after values.");
+
+      /* Skip opening brackets. */
+      values_start += 1;
+
+      StringRef enum_values = input.substr(values_start, values_end - values_start);
+
+      /* Really poor check. Could be done better. */
+      int64_t token = find_token(enum_values, '{');
+      int64_t not_found = (token == -1) ? 0 : -1;
+      CHECK(not_found, input, values_start + token, "Unexpected \'{\' token inside enum values.");
+
+      /* Do not capture the comma after the last value (if present). */
+      int64_t last_equal = rfind_token(enum_values, '=', values_end);
+      int64_t last_comma = rfind_token(enum_values, ',', values_end);
+      if (last_comma > last_equal) {
+        enum_values = input.substr(values_start, last_comma);
+      }
+
+      output += "const uint " + enum_values;
+
+      int64_t semicolon_found = (input[values_end + 1] == ';') ? 0 : -1;
+      CHECK(semicolon_found, input, values_end + 1, "Expected \';\' after enum type declaration.");
+
+      /* Skip the curly bracket but not the semicolon. */
+      cursor = last_pos = values_end + 1;
+    }
+    /* If nothing has been changed, do not allocate processed_source. */
+    if (last_pos == 0) {
+      return;
+    }
+
+#undef find_keyword
+#undef find_token
+#undef rfind_token
+
+    if (last_pos != 0) {
+      output += input.substr(last_pos);
+    }
+    processed_source = output;
+    source = processed_source.c_str();
   };
 
   void init_dependencies(const GPUSourceDictionnary &dict)
@@ -113,8 +304,8 @@ struct GPUSource {
       if (pos == -1) {
         return;
       }
-      int64_t start = source.find("(", pos) + 1;
-      int64_t end = source.find(")", pos);
+      int64_t start = source.find('(', pos) + 1;
+      int64_t end = source.find(')', pos);
       if (end == -1) {
         /* TODO Use clog. */
         std::cout << "Error: " << filename << " : Malformed BLENDER_REQUIRE: Missing \")\"."
@@ -161,8 +352,8 @@ void gpu_shader_dependency_init()
 {
   g_sources = new GPUSourceDictionnary();
 
-#define SHADER_SOURCE(datatoc, filename) \
-  g_sources->add_new(filename, new GPUSource(filename, datatoc));
+#define SHADER_SOURCE(datatoc, filename, filepath) \
+  g_sources->add_new(filename, new GPUSource(filepath, filename, datatoc));
 #include "glsl_draw_source_list.h"
 #include "glsl_gpu_source_list.h"
 #undef SHADER_SOURCE
diff --git a/source/blender/gpu/intern/gpu_shader_shared_utils.h b/source/blender/gpu/intern/gpu_shader_shared_utils.h
index f061f0f968a..9287011ddda 100644
--- a/source/blender/gpu/intern/gpu_shader_shared_utils.h
+++ b/source/blender/gpu/intern/gpu_shader_shared_utils.h
@@ -27,11 +27,10 @@
  * Some preprocessing is done by the GPU back-end to make it GLSL compatible.
  *
  * IMPORTANT:
- * - Don't add trailing comma at the end of the enum. Our custom pre-processor will now trim it
- *   for GLSL.
  * - Always use `u` suffix for enum values. GLSL do not support implicit cast.
  * - Define all values. This is in order to simplify custom pre-processor code.
- * - Always use uint32_t as underlying type.
+ * - (C++ only) Always use `uint32_t` as underlying type (`enum eMyEnum : uint32_t`).
+ * - (C only) do NOT use the enum type inside UBO/SSBO structs and use `uint` instead.
  * - Use float suffix by default for float literals to avoid double promotion in C++.
  * - Pack one float or int after a vec3/ivec3 to fulfill alignment rules.
  *