Update Export

leenkx/blender/lnx/assets.py

@@ -23,7 +23,15 @@ shaders_external = []
 shader_datas = []
 shader_passes = []
 shader_passes_assets = {}
-shader_cons = {}
+shader_cons = {
+    'mesh_vert': [], 'depth_vert': [], 'depth_frag': [],
+    'voxel_vert': [], 'voxel_frag': [], 'voxel_geom': [],
+}
+
+
+def add_world_def(wrd, define):
+    if define not in wrd.world_defs:
+        wrd.world_defs += define
 
 def reset():
     global assets
@@ -36,7 +44,6 @@ def reset():
     global shaders_external
     global shader_datas
     global shader_passes
-    global shader_cons
     assets = []
     khafile_params = []
     khafile_defs_last = khafile_defs
@@ -47,17 +54,10 @@ def reset():
     shaders_external = []
     shader_datas = []
     shader_passes = []
-    shader_cons = {}
+    reset_shader_cons()
-    shader_cons['mesh_vert'] = []
-    shader_cons['depth_vert'] = []
-    shader_cons['depth_frag'] = []
-    shader_cons['voxel_vert'] = []
-    shader_cons['voxel_frag'] = []
-    shader_cons['voxel_geom'] = []
 
 def reset_shader_cons():
     # Reset shader comparison arrays to prevent cross-scene shader merging
-    global shader_cons
     shader_cons['mesh_vert'] = []
     shader_cons['depth_vert'] = []
     shader_cons['depth_frag'] = []
@@ -66,8 +66,6 @@ def reset_shader_cons():
     shader_cons['voxel_geom'] = []
 
 def add(asset_file):
-    global assets
-
     # Asset already exists, do nothing
     if asset_file in assets:
         return

leenkx/blender/lnx/exporter.py

@@ -134,6 +134,9 @@ class LeenkxExporter:
     export_all_flag = True
     # Indicates whether rigid body is exported
     export_physics = False
+    export_navigation = False
+    export_ui = False
+    export_network = False
     optimize_enabled = False
     option_mesh_only = False
 

leenkx/blender/lnx/make.py

@@ -29,6 +29,8 @@ import lnx.make_logic as make_logic
 import lnx.make_renderpath as make_renderpath
 import lnx.make_state as state
 import lnx.make_world as make_world
+import lnx.material.make_shader as make_shader
+import lnx.material.mat_state as mat_state
 import lnx.utils
 import lnx.utils_vs
 import lnx.write_data as write_data
@@ -45,6 +47,7 @@ if lnx.is_reload(__name__):
     make_renderpath = lnx.reload_module(make_renderpath)
     state = lnx.reload_module(state)
     make_world = lnx.reload_module(make_world)
+    mat_state = lnx.reload_module(mat_state)
     lnx.utils = lnx.reload_module(lnx.utils)
     lnx.utils_vs = lnx.reload_module(lnx.utils_vs)
     write_data = lnx.reload_module(write_data)
@@ -238,6 +241,7 @@ def export_data_impl(fp, sdk_path):
     export_network = bpy.data.worlds['Lnx'].lnx_network != 'Disabled'
 
     assets.reset()
+    mat_state.material_cache.clear()
 
     # Build node trees
     LeenkxExporter.import_traits = []
@@ -256,6 +260,13 @@ def export_data_impl(fp, sdk_path):
     if not os.path.exists(build_dir + '/compiled/Assets'):
         os.makedirs(build_dir + '/compiled/Assets')
 
+    ext = '.lz4' if LeenkxExporter.compress_enabled else '.lnx'
+    scene_targets = []
+    for scene in bpy.data.scenes:
+        if scene.lnx_export:
+            asset_path = build_dir + '/compiled/Assets/' + lnx.utils.safestr(scene.name + "_" + os.path.basename(scene.library.filepath).replace(".blend", "") if scene.library else scene.name) + ext
+            scene_targets.append((scene.name, asset_path))
+
     # Make all 'MESH' and 'EMPTY' objects visible to the depsgraph (we pass
     # this to the exporter further below) with a temporary "zoo" collection
     # in the current scene. We do this to ensure that (among other things)
@@ -276,22 +287,19 @@ def export_data_impl(fp, sdk_path):
 
     build_cache = BuildExportCache()
 
-    for scene in bpy.data.scenes:
+    for scene_name, asset_path in scene_targets:
-        if scene.lnx_export:
+        # Reset shader comparison arrays to prevent cross-scene shader merging
-            # Reset shader comparison arrays to prevent cross-scene shader merging
+        assets.reset_shader_cons()
-            assets.reset_shader_cons()
+        LeenkxExporter.export_scene(bpy.context, asset_path, scene=bpy.data.scenes[scene_name], depsgraph=depsgraph, build_cache=build_cache)
-            ext = '.lz4' if LeenkxExporter.compress_enabled else '.lnx'
+        if LeenkxExporter.export_physics:
-            asset_path = build_dir + '/compiled/Assets/' + lnx.utils.safestr(scene.name + "_" + os.path.basename(scene.library.filepath).replace(".blend", "") if scene.library else scene.name) + ext
+            physics_found = True
-            LeenkxExporter.export_scene(bpy.context, asset_path, scene=scene, depsgraph=depsgraph, build_cache=build_cache)
+        if LeenkxExporter.export_navigation:
-            if LeenkxExporter.export_physics:
+            navigation_found = True
-                physics_found = True
+        if LeenkxExporter.export_ui:
-            if LeenkxExporter.export_navigation:
+            ui_found = True
-                navigation_found = True
+        if LeenkxExporter.export_network:
-            if LeenkxExporter.export_ui:
+            network_found = True
-                ui_found = True
+        assets.add(asset_path)
-            if LeenkxExporter.export_network:
-                network_found = True
-            assets.add(asset_path)
 
     if physics_found is False: # Disable physics if no rigid body is exported
         export_physics = False

leenkx/blender/lnx/make_renderpath.py

@@ -523,6 +523,9 @@ def build():
             elif node.type == 'SUBSURFACE_SCATTERING':
                 if '_SSS' not in wrd.world_defs:
                     wrd.world_defs += '_SSS'
+            elif node.type == 'EMISSION':
+                if '_EmissionShadeless' not in wrd.world_defs:
+                    wrd.world_defs += '_EmissionShadeless'
 
     gbuffer2 = '_Veloc' in wrd.world_defs or '_IgnoreIrr' in wrd.world_defs or '_VoxelGI' in wrd.world_defs or '_VoxelShadow' in wrd.world_defs or '_SSGI' in wrd.world_defs or '_Anisotropy' in wrd.world_defs
 

leenkx/blender/lnx/material/cycles_nodes/nodes_texture.py

@@ -378,7 +378,8 @@ def parse_sky_hosekwilkie(node: bpy.types.ShaderNodeTexSky, state: ParserState)
 
         # Radiance
         if rpdat.lnx_radiance and rpdat.lnx_irradiance and not mobile_mat:
-            wrd.world_defs += '_Rad'
+            if '_Rad' not in wrd.world_defs:
+                wrd.world_defs += '_Rad'
             assets.add_khafile_def("lnx_radiance")
             hosek_path = 'leenkx/Assets/hosek/'
             sdk_path = lnx.utils.get_sdk_path()
@@ -556,7 +557,8 @@ def parse_tex_environment(node: bpy.types.ShaderNodeTexEnvironment, out_socket:
 
     # Append radiance define
     if rpdat.lnx_irradiance and rpdat.lnx_radiance and not mobile_mat:
-        wrd.world_defs += '_Rad'
+        if '_Rad' not in wrd.world_defs:
+            wrd.world_defs += '_Rad'
         assets.add_khafile_def("lnx_radiance")
 
     return 'texture(envmap, envMapEquirect(pos)).rgb * envmapStrength'

leenkx/blender/lnx/material/make.py

@@ -4,6 +4,7 @@ import bpy
 from bpy.types import Material
 from bpy.types import Object
 
+import lnx.assets as assets
 import lnx.log as log
 import lnx.material.cycles as cycles
 import lnx.material.make_shader as make_shader
@@ -14,6 +15,7 @@ import lnx.node_utils
 import lnx.utils
 
 if lnx.is_reload(__name__):
+    assets = lnx.reload_module(assets)
     log = lnx.reload_module(log)
     cycles = lnx.reload_module(cycles)
     make_shader = lnx.reload_module(make_shader)

leenkx/blender/lnx/material/make_shader.py

@@ -1,5 +1,7 @@
 import os
+import re
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, List, Tuple
 
 import bpy
@@ -46,6 +48,32 @@ else:
 rpass_hook = None
 
 
+def cache_hit(cached, wrd, full_path, matname):
+    mat_state.features = dict(cached['features'])
+    mat_state.emission_type = cached['emission_type']
+    mat_state.texture_grad = cached['texture_grad']
+    mat_state.data = cached['shader_data']
+
+    if cached['world_defs_delta']:
+        for define in re.findall(r'_[A-Za-z0-9]+', cached['world_defs_delta']):
+            assets.add_world_def(wrd, define)
+
+    for d in cached['khafile_defs_delta']:
+        assets.add_khafile_def(d)
+
+    for shader_path in cached['shader_paths']:
+        assets.add_shader(shader_path)
+
+    shader_data_name = cached['shader_data_name']
+    if wrd.lnx_single_data_file:
+        pass
+    else:
+        shader_data_path = lnx.utils.get_fp_build() + '/compiled/Shaders/' + shader_data_name + '.lnx'
+        assets.add_shader_data(shader_data_path)
+
+    return cached['rpasses'], mat_state.data, shader_data_name, cached['bind_constants'], cached['bind_textures']
+
+
 def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxusers) -> Tuple:
     mat_state.mat_users = mat_users
     mat_state.mat_lnxusers = mat_lnxusers
@@ -68,8 +96,22 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus
 
     make_instancing_and_skinning(material, mat_users)
 
+    cache_key = None
+    cached = None
+    if material.signature:
+        global_elems_key = tuple((e['name'], e['data']) for e in mat_state.data.global_elems)
+        cache_key = (material.signature, mat_state.uses_instancing, tuple(rpasses), global_elems_key)
+        cached = mat_state.material_cache.get(cache_key)
+
+    if cached is not None:
+        return cache_hit(cached, wrd, full_path, matname)
+
+    world_defs_before = wrd.world_defs
+    khafile_defs_before = set(assets.khafile_defs)
+
     bind_constants = dict()
     bind_textures = dict()
+    all_shader_paths = []
 
     for rp in rpasses:
         car = []
@@ -117,7 +159,9 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus
         elif rpass_hook is not None:
             con = rpass_hook(rp)
 
-        write_shaders(rel_path, con, rp, matname)
+        if con is not None:
+            rp_shader_paths = write_shaders(rel_path, con, rp, matname)
+            all_shader_paths.extend(rp_shader_paths)
 
     shader_data_name = matname + '_data'
 
@@ -130,16 +174,85 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus
         shader_data_path = lnx.utils.get_fp_build() + '/compiled/Shaders/' + shader_data_name + '.lnx'
         assets.add_shader_data(shader_data_path)
 
+    if cache_key is not None:
+        world_defs_delta = wrd.world_defs[len(world_defs_before):]
+        khafile_defs_delta = [d for d in assets.khafile_defs if d not in khafile_defs_before]
+        mat_state.material_cache[cache_key] = {
+            'shader_data': mat_state.data,
+            'shader_data_name': shader_data_name,
+            'bind_constants': bind_constants,
+            'bind_textures': bind_textures,
+            'features': dict(mat_state.features),
+            'emission_type': mat_state.emission_type,
+            'texture_grad': mat_state.texture_grad,
+            'shader_paths': all_shader_paths,
+            'world_defs_delta': world_defs_delta,
+            'khafile_defs_delta': khafile_defs_delta,
+            'rpasses': rpasses,
+        }
+
     return rpasses, mat_state.data, shader_data_name, bind_constants, bind_textures
 
 
-def write_shaders(rel_path: str, con: ShaderContext, rpass: str, matname: str) -> None:
+def write_shaders(rel_path: str, con: ShaderContext, rpass: str, matname: str) -> List[str]:
     keep_cache = mat_state.material.lnx_cached
-    write_shader(rel_path, con.vert, 'vert', rpass, matname, keep_cache=keep_cache)
+    shaders = [con.vert, con.frag, con.geom, con.tesc, con.tese]
-    write_shader(rel_path, con.frag, 'frag', rpass, matname, keep_cache=keep_cache)
+    exts = ['vert', 'frag', 'geom', 'tesc', 'tese']
-    write_shader(rel_path, con.geom, 'geom', rpass, matname, keep_cache=keep_cache)
+    shader_paths = []
-    write_shader(rel_path, con.tesc, 'tesc', rpass, matname, keep_cache=keep_cache)
+
-    write_shader(rel_path, con.tese, 'tese', rpass, matname, keep_cache=keep_cache)
+    write_tasks = []
+    for shader, ext in zip(shaders, exts):
+        if shader is None or shader.is_linked:
+            continue
+
+        validation_issues = shader.validate()
+        if validation_issues:
+            for issue in validation_issues:
+                log.warn(f"Shader validation issue in {matname}_{rpass}.{ext}: {issue}. ")
+
+        output_rpass = rpass
+        if output_rpass == 'mesh' and mat_state.material.lnx_blending:
+            output_rpass = 'blend'
+
+        output_ext = '.glsl'
+        output_rel_path = rel_path
+        if shader.noprocessing:
+            hlsl_dir = lnx.utils.build_dir() + '/compiled/Hlsl/'
+            os.makedirs(hlsl_dir, exist_ok=True)
+            output_ext = '.hlsl'
+            output_rel_path = rel_path.replace('/compiled/Shaders/', '/compiled/Hlsl/')
+
+        shader_file = matname + '_' + output_rpass + '.' + ext + output_ext
+        shader_path = lnx.utils.get_fp() + '/' + output_rel_path + '/' + shader_file
+        assets.add_shader(shader_path)
+        shader_paths.append(shader_path)
+
+        if os.path.isfile(shader_path) and keep_cache:
+            continue
+
+        content = shader.get()
+
+        if shader.noprocessing:
+            written = lnx.utils.write_if_changed(shader_path, content)
+            if written:
+                cwd = os.getcwd()
+                os.chdir(lnx.utils.get_fp() + '/' + output_rel_path)
+                hlslbin_path = lnx.utils.get_sdk_path() + '/lib/leenkx_tools/hlslbin/hlslbin.exe'
+                prof = 'vs_5_0' if ext == 'vert' else 'ps_5_0' if ext == 'frag' else 'gs_5_0'
+                args = [hlslbin_path.replace('/', '\\').replace('\\\\', '\\'), shader_file, shader_file[:-4] + 'glsl', prof]
+                if ext == 'vert':
+                    args.append('-i')
+                    args.append('pos')
+                proc = subprocess.call(args)
+                os.chdir(cwd)
+        else:
+            write_tasks.append((shader_path, content))
+
+    if write_tasks:
+        with ThreadPoolExecutor(max_workers=min(5, len(write_tasks))) as pool:
+            list(pool.map(lambda t: lnx.utils.write_if_changed(t[0], t[1]), write_tasks))
+
+    return shader_paths
 
 
 def write_shader(rel_path: str, shader: Shader, ext: str, rpass: str, matname: str, keep_cache=True) -> None:
@@ -159,8 +272,7 @@ def write_shader(rel_path: str, shader: Shader, ext: str, rpass: str, matname: s
     if shader.noprocessing:
         # Use hlsl directly
         hlsl_dir = lnx.utils.build_dir() + '/compiled/Hlsl/'
-        if not os.path.exists(hlsl_dir):
+        os.makedirs(hlsl_dir, exist_ok=True)
-            os.makedirs(hlsl_dir)
         file_ext = '.hlsl'
         rel_path = rel_path.replace('/compiled/Shaders/', '/compiled/Hlsl/')
 

leenkx/blender/lnx/material/mat_state.py

@@ -40,3 +40,4 @@ uses_instancing = False  # Whether the current material has at least one user wi
 emission_type = EmissionType.NO_EMISSION
 features = {}  # tracks extended BRDF features used by current material
 next_ext_mat_id = 3  # auto assigned materialID for extended BRDF 
+material_cache = {}  # export material shader cache

leenkx/blender/lnx/utils.py

@@ -1064,7 +1064,13 @@ def def_strings_to_array(strdefs):
     defs = strdefs.split('_')
     defs = defs[1:]
     defs = ['_' + d for d in defs] # Restore _
-    return defs
+    seen = set()
+    result = []
+    for d in defs:
+        if d not in seen:
+            seen.add(d)
+            result.append(d)
+    return result
 
 def get_kha_target(target_name): # TODO: remove
     if target_name == 'osx-hl' or target_name == 'macos-hl':