diff --git a/leenkx/blender/lnx/assets.py b/leenkx/blender/lnx/assets.py index 14afa21a..81a70414 100644 --- a/leenkx/blender/lnx/assets.py +++ b/leenkx/blender/lnx/assets.py @@ -23,7 +23,15 @@ shaders_external = [] shader_datas = [] shader_passes = [] shader_passes_assets = {} -shader_cons = {} +shader_cons = { + 'mesh_vert': [], 'depth_vert': [], 'depth_frag': [], + 'voxel_vert': [], 'voxel_frag': [], 'voxel_geom': [], +} + + +def add_world_def(wrd, define): + if define not in wrd.world_defs: + wrd.world_defs += define def reset(): global assets @@ -36,7 +44,6 @@ def reset(): global shaders_external global shader_datas global shader_passes - global shader_cons assets = [] khafile_params = [] khafile_defs_last = khafile_defs @@ -47,17 +54,10 @@ def reset(): shaders_external = [] shader_datas = [] shader_passes = [] - shader_cons = {} - shader_cons['mesh_vert'] = [] - shader_cons['depth_vert'] = [] - shader_cons['depth_frag'] = [] - shader_cons['voxel_vert'] = [] - shader_cons['voxel_frag'] = [] - shader_cons['voxel_geom'] = [] + reset_shader_cons() def reset_shader_cons(): # Reset shader comparison arrays to prevent cross-scene shader merging - global shader_cons shader_cons['mesh_vert'] = [] shader_cons['depth_vert'] = [] shader_cons['depth_frag'] = [] @@ -66,8 +66,6 @@ def reset_shader_cons(): shader_cons['voxel_geom'] = [] def add(asset_file): - global assets - # Asset already exists, do nothing if asset_file in assets: return diff --git a/leenkx/blender/lnx/exporter.py b/leenkx/blender/lnx/exporter.py index f7d52b52..cc9f00ce 100644 --- a/leenkx/blender/lnx/exporter.py +++ b/leenkx/blender/lnx/exporter.py @@ -134,6 +134,9 @@ class LeenkxExporter: export_all_flag = True # Indicates whether rigid body is exported export_physics = False + export_navigation = False + export_ui = False + export_network = False optimize_enabled = False option_mesh_only = False diff --git a/leenkx/blender/lnx/make.py b/leenkx/blender/lnx/make.py index 59379146..8a2705a9 100644 --- a/leenkx/blender/lnx/make.py +++ b/leenkx/blender/lnx/make.py @@ -29,6 +29,8 @@ import lnx.make_logic as make_logic import lnx.make_renderpath as make_renderpath import lnx.make_state as state import lnx.make_world as make_world +import lnx.material.make_shader as make_shader +import lnx.material.mat_state as mat_state import lnx.utils import lnx.utils_vs import lnx.write_data as write_data @@ -45,6 +47,7 @@ if lnx.is_reload(__name__): make_renderpath = lnx.reload_module(make_renderpath) state = lnx.reload_module(state) make_world = lnx.reload_module(make_world) + mat_state = lnx.reload_module(mat_state) lnx.utils = lnx.reload_module(lnx.utils) lnx.utils_vs = lnx.reload_module(lnx.utils_vs) write_data = lnx.reload_module(write_data) @@ -238,6 +241,7 @@ def export_data_impl(fp, sdk_path): export_network = bpy.data.worlds['Lnx'].lnx_network != 'Disabled' assets.reset() + mat_state.material_cache.clear() # Build node trees LeenkxExporter.import_traits = [] @@ -256,6 +260,13 @@ def export_data_impl(fp, sdk_path): if not os.path.exists(build_dir + '/compiled/Assets'): os.makedirs(build_dir + '/compiled/Assets') + ext = '.lz4' if LeenkxExporter.compress_enabled else '.lnx' + scene_targets = [] + for scene in bpy.data.scenes: + if scene.lnx_export: + asset_path = build_dir + '/compiled/Assets/' + lnx.utils.safestr(scene.name + "_" + os.path.basename(scene.library.filepath).replace(".blend", "") if scene.library else scene.name) + ext + scene_targets.append((scene.name, asset_path)) + # Make all 'MESH' and 'EMPTY' objects visible to the depsgraph (we pass # this to the exporter further below) with a temporary "zoo" collection # in the current scene. We do this to ensure that (among other things) @@ -276,22 +287,19 @@ def export_data_impl(fp, sdk_path): build_cache = BuildExportCache() - for scene in bpy.data.scenes: - if scene.lnx_export: - # Reset shader comparison arrays to prevent cross-scene shader merging - assets.reset_shader_cons() - ext = '.lz4' if LeenkxExporter.compress_enabled else '.lnx' - asset_path = build_dir + '/compiled/Assets/' + lnx.utils.safestr(scene.name + "_" + os.path.basename(scene.library.filepath).replace(".blend", "") if scene.library else scene.name) + ext - LeenkxExporter.export_scene(bpy.context, asset_path, scene=scene, depsgraph=depsgraph, build_cache=build_cache) - if LeenkxExporter.export_physics: - physics_found = True - if LeenkxExporter.export_navigation: - navigation_found = True - if LeenkxExporter.export_ui: - ui_found = True - if LeenkxExporter.export_network: - network_found = True - assets.add(asset_path) + for scene_name, asset_path in scene_targets: + # Reset shader comparison arrays to prevent cross-scene shader merging + assets.reset_shader_cons() + LeenkxExporter.export_scene(bpy.context, asset_path, scene=bpy.data.scenes[scene_name], depsgraph=depsgraph, build_cache=build_cache) + if LeenkxExporter.export_physics: + physics_found = True + if LeenkxExporter.export_navigation: + navigation_found = True + if LeenkxExporter.export_ui: + ui_found = True + if LeenkxExporter.export_network: + network_found = True + assets.add(asset_path) if physics_found is False: # Disable physics if no rigid body is exported export_physics = False diff --git a/leenkx/blender/lnx/make_renderpath.py b/leenkx/blender/lnx/make_renderpath.py index e7362538..281796da 100644 --- a/leenkx/blender/lnx/make_renderpath.py +++ b/leenkx/blender/lnx/make_renderpath.py @@ -523,6 +523,9 @@ def build(): elif node.type == 'SUBSURFACE_SCATTERING': if '_SSS' not in wrd.world_defs: wrd.world_defs += '_SSS' + elif node.type == 'EMISSION': + if '_EmissionShadeless' not in wrd.world_defs: + wrd.world_defs += '_EmissionShadeless' gbuffer2 = '_Veloc' in wrd.world_defs or '_IgnoreIrr' in wrd.world_defs or '_VoxelGI' in wrd.world_defs or '_VoxelShadow' in wrd.world_defs or '_SSGI' in wrd.world_defs or '_Anisotropy' in wrd.world_defs diff --git a/leenkx/blender/lnx/material/cycles_nodes/nodes_texture.py b/leenkx/blender/lnx/material/cycles_nodes/nodes_texture.py index 989842f7..f68d426f 100644 --- a/leenkx/blender/lnx/material/cycles_nodes/nodes_texture.py +++ b/leenkx/blender/lnx/material/cycles_nodes/nodes_texture.py @@ -378,7 +378,8 @@ def parse_sky_hosekwilkie(node: bpy.types.ShaderNodeTexSky, state: ParserState) # Radiance if rpdat.lnx_radiance and rpdat.lnx_irradiance and not mobile_mat: - wrd.world_defs += '_Rad' + if '_Rad' not in wrd.world_defs: + wrd.world_defs += '_Rad' assets.add_khafile_def("lnx_radiance") hosek_path = 'leenkx/Assets/hosek/' sdk_path = lnx.utils.get_sdk_path() @@ -556,7 +557,8 @@ def parse_tex_environment(node: bpy.types.ShaderNodeTexEnvironment, out_socket: # Append radiance define if rpdat.lnx_irradiance and rpdat.lnx_radiance and not mobile_mat: - wrd.world_defs += '_Rad' + if '_Rad' not in wrd.world_defs: + wrd.world_defs += '_Rad' assets.add_khafile_def("lnx_radiance") return 'texture(envmap, envMapEquirect(pos)).rgb * envmapStrength' diff --git a/leenkx/blender/lnx/material/make.py b/leenkx/blender/lnx/material/make.py index e13a49fa..7777ba80 100644 --- a/leenkx/blender/lnx/material/make.py +++ b/leenkx/blender/lnx/material/make.py @@ -4,6 +4,7 @@ import bpy from bpy.types import Material from bpy.types import Object +import lnx.assets as assets import lnx.log as log import lnx.material.cycles as cycles import lnx.material.make_shader as make_shader @@ -14,6 +15,7 @@ import lnx.node_utils import lnx.utils if lnx.is_reload(__name__): + assets = lnx.reload_module(assets) log = lnx.reload_module(log) cycles = lnx.reload_module(cycles) make_shader = lnx.reload_module(make_shader) diff --git a/leenkx/blender/lnx/material/make_shader.py b/leenkx/blender/lnx/material/make_shader.py index 2348dc86..0a6e9414 100644 --- a/leenkx/blender/lnx/material/make_shader.py +++ b/leenkx/blender/lnx/material/make_shader.py @@ -1,5 +1,7 @@ import os +import re import subprocess +from concurrent.futures import ThreadPoolExecutor from typing import Dict, List, Tuple import bpy @@ -46,6 +48,32 @@ else: rpass_hook = None +def cache_hit(cached, wrd, full_path, matname): + mat_state.features = dict(cached['features']) + mat_state.emission_type = cached['emission_type'] + mat_state.texture_grad = cached['texture_grad'] + mat_state.data = cached['shader_data'] + + if cached['world_defs_delta']: + for define in re.findall(r'_[A-Za-z0-9]+', cached['world_defs_delta']): + assets.add_world_def(wrd, define) + + for d in cached['khafile_defs_delta']: + assets.add_khafile_def(d) + + for shader_path in cached['shader_paths']: + assets.add_shader(shader_path) + + shader_data_name = cached['shader_data_name'] + if wrd.lnx_single_data_file: + pass + else: + shader_data_path = lnx.utils.get_fp_build() + '/compiled/Shaders/' + shader_data_name + '.lnx' + assets.add_shader_data(shader_data_path) + + return cached['rpasses'], mat_state.data, shader_data_name, cached['bind_constants'], cached['bind_textures'] + + def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxusers) -> Tuple: mat_state.mat_users = mat_users mat_state.mat_lnxusers = mat_lnxusers @@ -68,8 +96,22 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus make_instancing_and_skinning(material, mat_users) + cache_key = None + cached = None + if material.signature: + global_elems_key = tuple((e['name'], e['data']) for e in mat_state.data.global_elems) + cache_key = (material.signature, mat_state.uses_instancing, tuple(rpasses), global_elems_key) + cached = mat_state.material_cache.get(cache_key) + + if cached is not None: + return cache_hit(cached, wrd, full_path, matname) + + world_defs_before = wrd.world_defs + khafile_defs_before = set(assets.khafile_defs) + bind_constants = dict() bind_textures = dict() + all_shader_paths = [] for rp in rpasses: car = [] @@ -117,7 +159,9 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus elif rpass_hook is not None: con = rpass_hook(rp) - write_shaders(rel_path, con, rp, matname) + if con is not None: + rp_shader_paths = write_shaders(rel_path, con, rp, matname) + all_shader_paths.extend(rp_shader_paths) shader_data_name = matname + '_data' @@ -130,16 +174,85 @@ def build(material: Material, mat_users: Dict[Material, List[Object]], mat_lnxus shader_data_path = lnx.utils.get_fp_build() + '/compiled/Shaders/' + shader_data_name + '.lnx' assets.add_shader_data(shader_data_path) + if cache_key is not None: + world_defs_delta = wrd.world_defs[len(world_defs_before):] + khafile_defs_delta = [d for d in assets.khafile_defs if d not in khafile_defs_before] + mat_state.material_cache[cache_key] = { + 'shader_data': mat_state.data, + 'shader_data_name': shader_data_name, + 'bind_constants': bind_constants, + 'bind_textures': bind_textures, + 'features': dict(mat_state.features), + 'emission_type': mat_state.emission_type, + 'texture_grad': mat_state.texture_grad, + 'shader_paths': all_shader_paths, + 'world_defs_delta': world_defs_delta, + 'khafile_defs_delta': khafile_defs_delta, + 'rpasses': rpasses, + } + return rpasses, mat_state.data, shader_data_name, bind_constants, bind_textures -def write_shaders(rel_path: str, con: ShaderContext, rpass: str, matname: str) -> None: +def write_shaders(rel_path: str, con: ShaderContext, rpass: str, matname: str) -> List[str]: keep_cache = mat_state.material.lnx_cached - write_shader(rel_path, con.vert, 'vert', rpass, matname, keep_cache=keep_cache) - write_shader(rel_path, con.frag, 'frag', rpass, matname, keep_cache=keep_cache) - write_shader(rel_path, con.geom, 'geom', rpass, matname, keep_cache=keep_cache) - write_shader(rel_path, con.tesc, 'tesc', rpass, matname, keep_cache=keep_cache) - write_shader(rel_path, con.tese, 'tese', rpass, matname, keep_cache=keep_cache) + shaders = [con.vert, con.frag, con.geom, con.tesc, con.tese] + exts = ['vert', 'frag', 'geom', 'tesc', 'tese'] + shader_paths = [] + + write_tasks = [] + for shader, ext in zip(shaders, exts): + if shader is None or shader.is_linked: + continue + + validation_issues = shader.validate() + if validation_issues: + for issue in validation_issues: + log.warn(f"Shader validation issue in {matname}_{rpass}.{ext}: {issue}. ") + + output_rpass = rpass + if output_rpass == 'mesh' and mat_state.material.lnx_blending: + output_rpass = 'blend' + + output_ext = '.glsl' + output_rel_path = rel_path + if shader.noprocessing: + hlsl_dir = lnx.utils.build_dir() + '/compiled/Hlsl/' + os.makedirs(hlsl_dir, exist_ok=True) + output_ext = '.hlsl' + output_rel_path = rel_path.replace('/compiled/Shaders/', '/compiled/Hlsl/') + + shader_file = matname + '_' + output_rpass + '.' + ext + output_ext + shader_path = lnx.utils.get_fp() + '/' + output_rel_path + '/' + shader_file + assets.add_shader(shader_path) + shader_paths.append(shader_path) + + if os.path.isfile(shader_path) and keep_cache: + continue + + content = shader.get() + + if shader.noprocessing: + written = lnx.utils.write_if_changed(shader_path, content) + if written: + cwd = os.getcwd() + os.chdir(lnx.utils.get_fp() + '/' + output_rel_path) + hlslbin_path = lnx.utils.get_sdk_path() + '/lib/leenkx_tools/hlslbin/hlslbin.exe' + prof = 'vs_5_0' if ext == 'vert' else 'ps_5_0' if ext == 'frag' else 'gs_5_0' + args = [hlslbin_path.replace('/', '\\').replace('\\\\', '\\'), shader_file, shader_file[:-4] + 'glsl', prof] + if ext == 'vert': + args.append('-i') + args.append('pos') + proc = subprocess.call(args) + os.chdir(cwd) + else: + write_tasks.append((shader_path, content)) + + if write_tasks: + with ThreadPoolExecutor(max_workers=min(5, len(write_tasks))) as pool: + list(pool.map(lambda t: lnx.utils.write_if_changed(t[0], t[1]), write_tasks)) + + return shader_paths def write_shader(rel_path: str, shader: Shader, ext: str, rpass: str, matname: str, keep_cache=True) -> None: @@ -159,8 +272,7 @@ def write_shader(rel_path: str, shader: Shader, ext: str, rpass: str, matname: s if shader.noprocessing: # Use hlsl directly hlsl_dir = lnx.utils.build_dir() + '/compiled/Hlsl/' - if not os.path.exists(hlsl_dir): - os.makedirs(hlsl_dir) + os.makedirs(hlsl_dir, exist_ok=True) file_ext = '.hlsl' rel_path = rel_path.replace('/compiled/Shaders/', '/compiled/Hlsl/') diff --git a/leenkx/blender/lnx/material/mat_state.py b/leenkx/blender/lnx/material/mat_state.py index 3de76f86..3199673f 100644 --- a/leenkx/blender/lnx/material/mat_state.py +++ b/leenkx/blender/lnx/material/mat_state.py @@ -40,3 +40,4 @@ uses_instancing = False # Whether the current material has at least one user wi emission_type = EmissionType.NO_EMISSION features = {} # tracks extended BRDF features used by current material next_ext_mat_id = 3 # auto assigned materialID for extended BRDF +material_cache = {} # export material shader cache diff --git a/leenkx/blender/lnx/utils.py b/leenkx/blender/lnx/utils.py index 4498812d..c204cd02 100644 --- a/leenkx/blender/lnx/utils.py +++ b/leenkx/blender/lnx/utils.py @@ -1064,7 +1064,13 @@ def def_strings_to_array(strdefs): defs = strdefs.split('_') defs = defs[1:] defs = ['_' + d for d in defs] # Restore _ - return defs + seen = set() + result = [] + for d in defs: + if d not in seen: + seen.add(d) + result.append(d) + return result def get_kha_target(target_name): # TODO: remove if target_name == 'osx-hl' or target_name == 'macos-hl':