From 01637056405df47b072bf9b3aee5a34d9dff35e8 Mon Sep 17 00:00:00 2001 From: Thaddeus Crews Date: Wed, 31 Jul 2024 14:15:56 -0500 Subject: [PATCH] SCons: Add method to generate raw cstrings --- editor/icons/editor_icons_builders.py | 20 +++------ gles3_builders.py | 18 +++----- glsl_builders.py | 28 +++---------- methods.py | 42 ++++++++++++++++++- platform/SCsub | 6 +-- .../icons/default_theme_icons_builders.py | 20 +++------ .../gles3/vertex_fragment_expected_full.glsl | 28 ++++++++++++- .../fixtures/glsl/compute_expected_full.glsl | 14 ++++++- .../glsl/vertex_fragment_expected_full.glsl | 34 ++++++++++++++- .../rd_glsl/compute_expected_full.glsl | 14 ++++++- .../vertex_fragment_expected_full.glsl | 27 +++++++++++- 11 files changed, 175 insertions(+), 76 deletions(-) diff --git a/editor/icons/editor_icons_builders.py b/editor/icons/editor_icons_builders.py index 5cc67ca3ad7..d3e89534835 100644 --- a/editor/icons/editor_icons_builders.py +++ b/editor/icons/editor_icons_builders.py @@ -3,6 +3,8 @@ import os from io import StringIO +from methods import to_raw_cstring + # See also `scene/theme/icons/default_theme_icons_builders.py`. def make_editor_icons_action(target, source, env): @@ -10,21 +12,9 @@ def make_editor_icons_action(target, source, env): svg_icons = source with StringIO() as icons_string, StringIO() as s: - for f in svg_icons: - fname = str(f) - - icons_string.write('\t"') - - with open(fname, "rb") as svgf: - b = svgf.read(1) - while len(b) == 1: - icons_string.write("\\" + str(hex(ord(b)))[1:]) - b = svgf.read(1) - - icons_string.write('"') - if fname != svg_icons[-1]: - icons_string.write(",") - icons_string.write("\n") + for svg in svg_icons: + with open(str(svg), "r") as svgf: + icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read())) s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") s.write("#ifndef _EDITOR_ICONS_H\n") diff --git a/gles3_builders.py b/gles3_builders.py index a4928c81c5c..a81d42b42e2 100644 --- a/gles3_builders.py +++ b/gles3_builders.py @@ -3,7 +3,7 @@ import os.path from typing import Optional -from methods import print_error +from methods import print_error, to_raw_cstring class GLES3HeaderStruct: @@ -553,20 +553,12 @@ def build_gles3_header( fd.write("\t\tstatic const Feedback* _feedbacks=nullptr;\n") fd.write("\t\tstatic const char _vertex_code[]={\n") - for x in header_data.vertex_lines: - for c in x: - fd.write(str(ord(c)) + ",") - - fd.write(str(ord("\n")) + ",") - fd.write("\t\t0};\n\n") + fd.write(to_raw_cstring(header_data.vertex_lines)) + fd.write("\n\t\t};\n\n") fd.write("\t\tstatic const char _fragment_code[]={\n") - for x in header_data.fragment_lines: - for c in x: - fd.write(str(ord(c)) + ",") - - fd.write(str(ord("\n")) + ",") - fd.write("\t\t0};\n\n") + fd.write(to_raw_cstring(header_data.fragment_lines)) + fd.write("\n\t\t};\n\n") fd.write( '\t\t_setup(_vertex_code,_fragment_code,"' diff --git a/glsl_builders.py b/glsl_builders.py index 05aab3acbb3..82c15fc93be 100644 --- a/glsl_builders.py +++ b/glsl_builders.py @@ -1,25 +1,9 @@ """Functions used to generate source files during build time""" import os.path -from typing import Iterable, Optional +from typing import Optional -from methods import print_error - - -def generate_inline_code(input_lines: Iterable[str], insert_newline: bool = True): - """Take header data and generate inline code - - :param: input_lines: values for shared inline code - :return: str - generated inline value - """ - output = [] - for line in input_lines: - if line: - output.append(",".join(str(ord(c)) for c in line)) - if insert_newline: - output.append("%s" % ord("\n")) - output.append("0") - return ",".join(output) +from methods import print_error, to_raw_cstring class RDHeaderStruct: @@ -127,13 +111,13 @@ def build_rd_header( if header_data.compute_lines: body_parts = [ - "static const char _compute_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.compute_lines), + "static const char _compute_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.compute_lines), f'setup(nullptr, nullptr, _compute_code, "{out_file_class}");', ] else: body_parts = [ - "static const char _vertex_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.vertex_lines), - "static const char _fragment_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.fragment_lines), + "static const char _vertex_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.vertex_lines), + "static const char _fragment_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.fragment_lines), f'setup(_vertex_code, _fragment_code, nullptr, "{out_file_class}");', ] @@ -211,7 +195,7 @@ def build_raw_header( #define {out_file_ifdef}_RAW_H static const char {out_file_base}[] = {{ - {generate_inline_code(header_data.code, insert_newline=False)} +{to_raw_cstring(header_data.code)} }}; #endif """ diff --git a/methods.py b/methods.py index c725501fd99..16ef0e126ed 100644 --- a/methods.py +++ b/methods.py @@ -8,7 +8,7 @@ from collections import OrderedDict from enum import Enum from io import StringIO, TextIOWrapper from pathlib import Path -from typing import Generator, Optional +from typing import Generator, List, Optional, Union # Get the "Godot" folder name ahead of time base_folder_path = str(os.path.abspath(Path(__file__).parent)) + "/" @@ -1641,3 +1641,43 @@ def generated_wrapper( file.write(f"\n\n#endif // {header_guard}") file.write("\n") + + +def to_raw_cstring(value: Union[str, List[str]]) -> str: + MAX_LITERAL = 16 * 1024 + + if isinstance(value, list): + value = "\n".join(value) + "\n" + + split: List[bytes] = [] + offset = 0 + encoded = value.encode() + + while offset <= len(encoded): + segment = encoded[offset : offset + MAX_LITERAL] + offset += MAX_LITERAL + if len(segment) == MAX_LITERAL: + # Try to segment raw strings at double newlines to keep readable. + pretty_break = segment.rfind(b"\n\n") + if pretty_break != -1: + segment = segment[: pretty_break + 1] + offset -= MAX_LITERAL - pretty_break - 1 + # If none found, ensure we end with valid utf8. + # https://github.com/halloleo/unicut/blob/master/truncate.py + elif segment[-1] & 0b10000000: + last_11xxxxxx_index = [i for i in range(-1, -5, -1) if segment[i] & 0b11000000 == 0b11000000][0] + last_11xxxxxx = segment[last_11xxxxxx_index] + if not last_11xxxxxx & 0b00100000: + last_char_length = 2 + elif not last_11xxxxxx & 0b0010000: + last_char_length = 3 + elif not last_11xxxxxx & 0b0001000: + last_char_length = 4 + + if last_char_length > -last_11xxxxxx_index: + segment = segment[:last_11xxxxxx_index] + offset += last_11xxxxxx_index + + split += [segment] + + return " ".join(f'R"({x.decode()})"' for x in split) diff --git a/platform/SCsub b/platform/SCsub index b07023efedc..cdaa6074baf 100644 --- a/platform/SCsub +++ b/platform/SCsub @@ -15,12 +15,12 @@ def export_icon_builder(target, source, env): src_path = Path(str(source[0])) src_name = src_path.stem platform = src_path.parent.parent.stem - with open(str(source[0]), "rb") as file: - svg = "".join([f"\\{hex(x)[1:]}" for x in file.read()]) + with open(str(source[0]), "r") as file: + svg = file.read() with methods.generated_wrapper(target, prefix=platform) as file: file.write( f"""\ -static const char *_{platform}_{src_name}_svg = "{svg}"; +static const char *_{platform}_{src_name}_svg = {methods.to_raw_cstring(svg)}; """ ) diff --git a/scene/theme/icons/default_theme_icons_builders.py b/scene/theme/icons/default_theme_icons_builders.py index 49c0a931915..3a673af92eb 100644 --- a/scene/theme/icons/default_theme_icons_builders.py +++ b/scene/theme/icons/default_theme_icons_builders.py @@ -3,6 +3,8 @@ import os from io import StringIO +from methods import to_raw_cstring + # See also `editor/icons/editor_icons_builders.py`. def make_default_theme_icons_action(target, source, env): @@ -10,21 +12,9 @@ def make_default_theme_icons_action(target, source, env): svg_icons = [str(x) for x in source] with StringIO() as icons_string, StringIO() as s: - for f in svg_icons: - fname = str(f) - - icons_string.write('\t"') - - with open(fname, "rb") as svgf: - b = svgf.read(1) - while len(b) == 1: - icons_string.write("\\" + str(hex(ord(b)))[1:]) - b = svgf.read(1) - - icons_string.write('"') - if fname != svg_icons[-1]: - icons_string.write(",") - icons_string.write("\n") + for svg in svg_icons: + with open(svg, "r") as svgf: + icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read())) s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n") s.write('#include "modules/modules_enabled.gen.h"\n\n') diff --git a/tests/python_build/fixtures/gles3/vertex_fragment_expected_full.glsl b/tests/python_build/fixtures/gles3/vertex_fragment_expected_full.glsl index 8ad5a23eb5b..db5f54e3d8b 100644 --- a/tests/python_build/fixtures/gles3/vertex_fragment_expected_full.glsl +++ b/tests/python_build/fixtures/gles3/vertex_fragment_expected_full.glsl @@ -37,10 +37,34 @@ protected: static const Feedback* _feedbacks=nullptr; static const char _vertex_code[]={ -10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,105,110,32,104,105,103,104,112,32,118,101,99,51,32,118,101,114,116,101,120,59,10,10,111,117,116,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,32,61,32,118,101,99,52,40,118,101,114,116,101,120,46,120,44,49,44,48,44,49,41,59,10,125,10,10, 0}; +R"( +precision highp float; +precision highp int; + +layout(location = 0) in highp vec3 vertex; + +out highp vec4 position_interp; + +void main() { + position_interp = vec4(vertex.x,1,0,1); +} + +)" + }; static const char _fragment_code[]={ -10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,105,110,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,104,105,103,104,112,32,102,108,111,97,116,32,100,101,112,116,104,32,61,32,40,40,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,122,32,47,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,119,41,32,43,32,49,46,48,41,59,10,9,102,114,97,103,95,99,111,108,111,114,32,61,32,118,101,99,52,40,100,101,112,116,104,41,59,10,125,10, 0}; +R"( +precision highp float; +precision highp int; + +in highp vec4 position_interp; + +void main() { + highp float depth = ((position_interp.z / position_interp.w) + 1.0); + frag_color = vec4(depth); +} +)" + }; _setup(_vertex_code,_fragment_code,"VertexFragmentShaderGLES3",0,_uniform_strings,0,_ubo_pairs,0,_feedbacks,0,_texunit_pairs,1,_spec_pairs,1,_variant_defines); } diff --git a/tests/python_build/fixtures/glsl/compute_expected_full.glsl b/tests/python_build/fixtures/glsl/compute_expected_full.glsl index b937d732c80..386d14f1aa2 100644 --- a/tests/python_build/fixtures/glsl/compute_expected_full.glsl +++ b/tests/python_build/fixtures/glsl/compute_expected_full.glsl @@ -3,6 +3,18 @@ #define COMPUTE_SHADER_GLSL_RAW_H static const char compute_shader_glsl[] = { - 35,91,99,111,109,112,117,116,101,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,118,101,99,51,32,115,116,97,116,105,99,95,108,105,103,104,116,32,61,32,118,101,99,51,40,48,44,32,49,44,32,48,41,59,10,125,10,0 +R"(#[compute] + +#version 450 + +#VERSION_DEFINES + + +#define M_PI 3.14159265359 + +void main() { + vec3 static_light = vec3(0, 1, 0); +} +)" }; #endif diff --git a/tests/python_build/fixtures/glsl/vertex_fragment_expected_full.glsl b/tests/python_build/fixtures/glsl/vertex_fragment_expected_full.glsl index 3f53a17fac2..b7329b6a796 100644 --- a/tests/python_build/fixtures/glsl/vertex_fragment_expected_full.glsl +++ b/tests/python_build/fixtures/glsl/vertex_fragment_expected_full.glsl @@ -3,6 +3,38 @@ #define VERTEX_FRAGMENT_SHADER_GLSL_RAW_H static const char vertex_fragment_shader_glsl[] = { - 35,91,118,101,114,115,105,111,110,115,93,10,10,108,105,110,101,115,32,61,32,34,35,100,101,102,105,110,101,32,77,79,68,69,95,76,73,78,69,83,34,59,10,10,35,91,118,101,114,116,101,120,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,51,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,10,35,105,102,100,101,102,32,77,79,68,69,95,76,73,78,69,83,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,51,40,48,44,48,44,49,41,59,10,35,101,110,100,105,102,10,125,10,10,35,91,102,114,97,103,109,101,110,116,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,52,32,100,115,116,95,99,111,108,111,114,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,100,115,116,95,99,111,108,111,114,32,61,32,118,101,99,52,40,49,44,49,44,48,44,48,41,59,10,125,10,0 +R"(#[versions] + +lines = "#define MODE_LINES"; + +#[vertex] + +#version 450 + +#VERSION_DEFINES + +layout(location = 0) out vec3 uv_interp; + +void main() { + +#ifdef MODE_LINES + uv_interp = vec3(0,0,1); +#endif +} + +#[fragment] + +#version 450 + +#VERSION_DEFINES + +#define M_PI 3.14159265359 + +layout(location = 0) out vec4 dst_color; + +void main() { + dst_color = vec4(1,1,0,0); +} +)" }; #endif diff --git a/tests/python_build/fixtures/rd_glsl/compute_expected_full.glsl b/tests/python_build/fixtures/rd_glsl/compute_expected_full.glsl index b59923e28a5..1184510020e 100644 --- a/tests/python_build/fixtures/rd_glsl/compute_expected_full.glsl +++ b/tests/python_build/fixtures/rd_glsl/compute_expected_full.glsl @@ -11,7 +11,19 @@ public: ComputeShaderRD() { static const char _compute_code[] = { -10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,66,76,79,67,75,95,83,73,90,69,32,56,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,105,110,116,32,116,32,61,32,66,76,79,67,75,95,83,73,90,69,32,43,32,49,59,10,125,10,0 +R"( +#version 450 + +#VERSION_DEFINES + +#define BLOCK_SIZE 8 + +#define M_PI 3.14159265359 + +void main() { + uint t = BLOCK_SIZE + 1; +} +)" }; setup(nullptr, nullptr, _compute_code, "ComputeShaderRD"); } diff --git a/tests/python_build/fixtures/rd_glsl/vertex_fragment_expected_full.glsl b/tests/python_build/fixtures/rd_glsl/vertex_fragment_expected_full.glsl index ff804dbf890..2f809f1bfe8 100644 --- a/tests/python_build/fixtures/rd_glsl/vertex_fragment_expected_full.glsl +++ b/tests/python_build/fixtures/rd_glsl/vertex_fragment_expected_full.glsl @@ -11,10 +11,33 @@ public: VertexFragmentShaderRD() { static const char _vertex_code[] = { -10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,50,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,50,40,48,44,32,49,41,59,10,125,10,10,0 +R"( +#version 450 + +#VERSION_DEFINES + +#define M_PI 3.14159265359 + +layout(location = 0) out vec2 uv_interp; + +void main() { + uv_interp = vec2(0, 1); +} + +)" }; static const char _fragment_code[] = { -10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,105,110,32,118,101,99,50,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,50,40,49,44,32,48,41,59,10,125,10,0 +R"( +#version 450 + +#VERSION_DEFINES + +layout(location = 0) in vec2 uv_interp; + +void main() { + uv_interp = vec2(1, 0); +} +)" }; setup(_vertex_code, _fragment_code, nullptr, "VertexFragmentShaderRD"); }