From 059638e6e018d9f0af21fd69859c10a1c51b6c53 Mon Sep 17 00:00:00 2001 From: max Date: Tue, 28 Apr 2026 19:17:23 +0200 Subject: [PATCH] resource stuff --- Nerfed.Builder/Builder/Builder.cs | 44 +++- Nerfed.Builder/Meta/AssetMeta.cs | 30 +++ Nerfed.Runtime/Graphics/GraphicsDevice.cs | 16 +- Nerfed.Runtime/Gui/GuiController.cs | 8 +- Nerfed.Runtime/Resource/Resource.cs | 22 ++ Nerfed.Runtime/Resource/ResourceComponents.cs | 19 ++ Nerfed.Runtime/Resource/ResourceManager.cs | 220 +++++++++++++++--- .../Resource/SampleMeshVisualComponent.cs | 19 ++ Nerfed.Runtime/Resource/SampleRenderSystem.cs | 53 +++++ 9 files changed, 386 insertions(+), 45 deletions(-) create mode 100644 Nerfed.Builder/Meta/AssetMeta.cs create mode 100644 Nerfed.Runtime/Resource/ResourceComponents.cs create mode 100644 Nerfed.Runtime/Resource/SampleMeshVisualComponent.cs create mode 100644 Nerfed.Runtime/Resource/SampleRenderSystem.cs diff --git a/Nerfed.Builder/Builder/Builder.cs b/Nerfed.Builder/Builder/Builder.cs index cee3206..5022ad8 100644 --- a/Nerfed.Builder/Builder/Builder.cs +++ b/Nerfed.Builder/Builder/Builder.cs @@ -1,4 +1,6 @@ using System.Diagnostics; +using System.Text.Json; +using Nerfed.Builder.Meta; namespace Nerfed.Builder; @@ -57,15 +59,45 @@ public class Builder : IDisposable string outFile = $"{args.ResourceOutPath}/{relativeFile}{PathUtil.ImportedFileExtension}"; FileInfo inFileInfo = new FileInfo(inFile); - FileInfo outFileInfo = new FileInfo(outFile); - if (!FileUtil.IsNewer(inFileInfo, outFileInfo)) + // ========================================================================= + // STEP 1: GUID META FILE SYNC + // Ensure the source file has a backing .meta file generating its Guid + // ========================================================================= + string metaFile = inFile + ".meta"; + AssetMeta metaData; + + if (!File.Exists(metaFile)) + { + // Generate a brand new meta file to track this asset permanently + metaData = new AssetMeta(Guid.NewGuid()); + string json = JsonSerializer.Serialize(metaData, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(metaFile, json); + Console.WriteLine($"[Meta] Generated new tracking ID '{metaData.Id}' for {relativeFile}"); + } + else + { + // Load the existing guid + metaData = JsonSerializer.Deserialize(File.ReadAllText(metaFile))!; + } + + // Change output file from Name.ext.bin -> /GUID.bin to completely anonymize the actual game package! + string cacheOutFile = $"{args.ResourceOutPath}/{metaData.Id}.bin"; + FileInfo outFileInfo = new FileInfo(cacheOutFile); + + // Rebuild if the source file changed, or if the meta file changed! + FileInfo metaFileInfo = new FileInfo(metaFile); + bool requiresCompile = !outFileInfo.Exists || + FileUtil.IsNewer(inFileInfo, outFileInfo) || + FileUtil.IsNewer(metaFileInfo, outFileInfo); + + if (!requiresCompile) { // File has not changed since last build, no need to build this one. return; } - string outDir = Path.GetDirectoryName(outFile); + string outDir = Path.GetDirectoryName(cacheOutFile)!; if (!Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); @@ -74,14 +106,14 @@ public class Builder : IDisposable string ext = Path.GetExtension(inFile).ToLower(); if (importers.TryGetValue(ext, out IImporter importer)) { - importer.Import(inFile, outFile); + importer.Import(inFile, cacheOutFile); // Compile source directly to hash.bin } else { - rawFileImporter.Import(inFile, outFile); + rawFileImporter.Import(inFile, cacheOutFile); } - Console.WriteLine(relativeFile); + Console.WriteLine($"Compiled {relativeFile} -> {metaData.Id}.bin"); } catch (Exception e) { diff --git a/Nerfed.Builder/Meta/AssetMeta.cs b/Nerfed.Builder/Meta/AssetMeta.cs new file mode 100644 index 0000000..a592643 --- /dev/null +++ b/Nerfed.Builder/Meta/AssetMeta.cs @@ -0,0 +1,30 @@ + using System; + + namespace Nerfed.Builder.Meta + { + /// + /// Foundation for JSON-serialized metadata files (e.g. hero.png.meta) + /// + public class AssetMeta + { + /// + /// The universally unique identifier for this asset, generated on first import. + /// + public Guid Id { get; set; } + + /// + /// The importer version. Useful to force re-imports if your engine updates how it parses textures. + /// + public int ImporterVersion { get; set; } = 1; + + /// + /// Base constructor needed for JSON deserialization + /// + public AssetMeta() { } + + public AssetMeta(Guid id) + { + Id = id; + } + } + } diff --git a/Nerfed.Runtime/Graphics/GraphicsDevice.cs b/Nerfed.Runtime/Graphics/GraphicsDevice.cs index f274e00..89ab7ca 100644 --- a/Nerfed.Runtime/Graphics/GraphicsDevice.cs +++ b/Nerfed.Runtime/Graphics/GraphicsDevice.cs @@ -68,10 +68,10 @@ public class GraphicsDevice : IDisposable internal void LoadDefaultPipelines() { - FullscreenVertexShader = ResourceManager.Load("Shaders/Fullscreen.vert"); - VideoFragmentShader = ResourceManager.Load("Shaders/Video.frag"); - TextVertexShader = ResourceManager.Load("Shaders/Text.vert"); - TextFragmentShader = ResourceManager.Load("Shaders/Text.frag"); + FullscreenVertexShader = ResourceManager.Retain("Shaders/Fullscreen.vert"); + VideoFragmentShader = ResourceManager.Retain("Shaders/Video.frag"); + TextVertexShader = ResourceManager.Retain("Shaders/Text.vert"); + TextFragmentShader = ResourceManager.Retain("Shaders/Text.frag"); VideoPipeline = new GraphicsPipeline( this, @@ -373,10 +373,10 @@ public class GraphicsDevice : IDisposable resources.Clear(); } - ResourceManager.Unload(FullscreenVertexShader); - ResourceManager.Unload(TextFragmentShader); - ResourceManager.Unload(TextVertexShader); - ResourceManager.Unload(VideoFragmentShader); + ResourceManager.Release(FullscreenVertexShader); + ResourceManager.Release(TextFragmentShader); + ResourceManager.Release(TextVertexShader); + ResourceManager.Release(VideoFragmentShader); } Refresh.Refresh_DestroyDevice(Handle); diff --git a/Nerfed.Runtime/Gui/GuiController.cs b/Nerfed.Runtime/Gui/GuiController.cs index a434211..7eaabd2 100644 --- a/Nerfed.Runtime/Gui/GuiController.cs +++ b/Nerfed.Runtime/Gui/GuiController.cs @@ -60,8 +60,8 @@ public class GuiController : IDisposable io.DisplaySize = new Vector2(mainWindow.Width, mainWindow.Height); io.DisplayFramebufferScale = Vector2.One; - imGuiVertexShader = ResourceManager.Load("Shaders/ImGui.vert"); - imGuiFragmentShader = ResourceManager.Load("Shaders/ImGui.frag"); + imGuiVertexShader = ResourceManager.Retain("Shaders/ImGui.vert"); + imGuiFragmentShader = ResourceManager.Retain("Shaders/ImGui.frag"); imGuiSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp); @@ -630,8 +630,8 @@ public class GuiController : IDisposable fontTexture?.Dispose(); imGuiVertexBuffer?.Dispose(); imGuiIndexBuffer?.Dispose(); - ResourceManager.Unload(imGuiVertexShader); - ResourceManager.Unload(imGuiFragmentShader); + ResourceManager.Release(imGuiVertexShader); + ResourceManager.Release(imGuiFragmentShader); imGuiPipeline?.Dispose(); imGuiSampler?.Dispose(); resourceUploader?.Dispose(); diff --git a/Nerfed.Runtime/Resource/Resource.cs b/Nerfed.Runtime/Resource/Resource.cs index 5a4a545..72d571b 100644 --- a/Nerfed.Runtime/Resource/Resource.cs +++ b/Nerfed.Runtime/Resource/Resource.cs @@ -1,8 +1,30 @@ +using System; namespace Nerfed.Runtime; +public enum ResourceState +{ + Unloaded, + Queued, + Loading, + Loaded, + Failed +} + public abstract class Resource { + public Guid Id { get; internal set; } public string Path { get; internal set; } + + /// + /// Natively tracks if the resource is currently in RAM/VRAM. + /// + public ResourceState State { get; internal set; } = ResourceState.Unloaded; + + /// + /// Tracks how many entities or systems currently need this loaded. + /// When it hits 0, the ResourceManager handles unloading natively. + /// + public int ReferenceCount { get; internal set; } = 0; internal abstract void Load(Stream stream); internal abstract void Unload(); diff --git a/Nerfed.Runtime/Resource/ResourceComponents.cs b/Nerfed.Runtime/Resource/ResourceComponents.cs new file mode 100644 index 0000000..671a311 --- /dev/null +++ b/Nerfed.Runtime/Resource/ResourceComponents.cs @@ -0,0 +1,19 @@ +namespace Nerfed.Runtime; + +/// +/// Attach this component to an entity mapped to raw source-path strings. +/// Useful for testing, hardcoded assets, or before full editor-guided GUID injection. +/// +public readonly record struct AssetReferenceComponent(Guid AssetId); + +/// +/// A strongly-typed version of an asset reference, preventing the user from accidentally +/// assigning a Shader GUID to a Texture component in the Editor. +/// +public readonly record struct TypedAssetReference(Guid AssetId) where TRes : Resource; + +/// +/// Added to an entity by the AssetStreamingSystem when the physical resource is fully +/// loaded in memory and ready to be used by the renderer or physics engine. +/// +public struct AssetLoadedTag { } diff --git a/Nerfed.Runtime/Resource/ResourceManager.cs b/Nerfed.Runtime/Resource/ResourceManager.cs index 2af777a..54fc4d1 100644 --- a/Nerfed.Runtime/Resource/ResourceManager.cs +++ b/Nerfed.Runtime/Resource/ResourceManager.cs @@ -1,43 +1,209 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Threading; + namespace Nerfed.Runtime; +/// +/// A highly scalable, multithreaded resource manager that handles asynchronous asset +/// loading and automatic reference-counted memory management. +/// public static class ResourceManager { - private const string rootName = "Resources"; - private static readonly Dictionary loadedResources = new Dictionary(); + private const string RootName = "Resources"; - public static T Load(string resourcePath) where T : Resource + // Track resources by their Guid ID instead of simple strings. + private static readonly ConcurrentDictionary _resourceCache = new(); + + // Mapping a string path to its runtime Guid identifier + private static readonly ConcurrentDictionary _pathToGuid = new(); + + // Queues for background processing + private static readonly ConcurrentQueue _loadQueue = new(); + + // Loader threads + private static readonly Thread _loaderThread; + private static bool _isRunning = true; + + // A registry of how to create concrete Resource instances from a generic type without massive switch statements. + private static readonly Dictionary> _resourceFactories = new() { - if (loadedResources.TryGetValue(resourcePath, out Resource resource)) + { typeof(Shader), () => new Shader() } + }; + + static ResourceManager() + { + _loaderThread = new Thread(LoaderWorkerLoop) { - return (T)resource; + Name = "Nerfed Asset Loader", + IsBackground = true, + Priority = ThreadPriority.BelowNormal // Keeps CPU time focused on the main game loop + }; + _loaderThread.Start(); + } + + /// + /// Synchronously shuts down the loader thread when the engine closes. + /// + public static void Shutdown() + { + _isRunning = false; + _loaderThread.Join(); + } + + /// + /// Registers a new resource type factory so the manager knows how to instantiate it. + /// Example: RegisterResourceType(() => new Texture()); + /// + public static void RegisterResourceType(Func factory) where T : Resource + { + _resourceFactories[typeof(T)] = factory; + } + + /// + /// Gets the Guid associated with a specific asset path, making an initial id pass if required. + /// In a fully baked engine, the Guid is known at compile time or baked in the map data. + /// + public static Guid GetId(string resourcePath) + { + return _pathToGuid.GetOrAdd(resourcePath, _ => Guid.NewGuid()); + } + + /// + /// Begins an asynchronous load for a resource by its Guid. + /// In ECS systems, Entities should strictly prefer this overload over the string one. + /// + public static T Retain(Guid id, string expectedPath) where T : Resource + { + var resource = _resourceCache.GetOrAdd(id, (assetId) => + { + if (!_resourceFactories.TryGetValue(typeof(T), out var factory)) + { + throw new Exception($"Failed to create resource. No factory registered for {typeof(T).Name}"); + } + + var newResource = factory(); + newResource.Id = assetId; + // The path is still required so the background thread knows which file to open from disk. + newResource.Path = expectedPath; + newResource.State = ResourceState.Unloaded; + + return newResource; + }); + + lock (resource) + { + resource.ReferenceCount++; + + if (resource.State == ResourceState.Unloaded) + { + resource.State = ResourceState.Queued; + _loadQueue.Enqueue(resource); + } } - if (typeof(T) == typeof(Shader)) - { - resource = new Shader(); - } - else - { - throw new Exception("Failed to create resource"); - } - - Assert.Always(resource != null); - resource.Path = resourcePath; - resource.Load(StorageContainer.OpenStream(Path.Combine(AppContext.BaseDirectory, rootName, resourcePath) + ".bin")); - - loadedResources.Add(resourcePath, resource); return (T)resource; } - public static void Unload(Resource resource) + /// + /// Begins an asynchronous load utilizing the string path to find the matching Guid. + /// This should generally be avoided in tight ECS loops. + /// + public static T Retain(string resourcePath) where T : Resource { - if (!loadedResources.ContainsKey(resource.Path)) - { - return; - } + Guid id = GetId(resourcePath); + return Retain(id, resourcePath); + } - resource.Unload(); - resource.Path = string.Empty; - loadedResources.Remove(resource.Path); + /// + /// Gets the current loading state of a resource by its Guid without altering its reference count. + /// + public static ResourceState GetState(Guid id) + { + if (_resourceCache.TryGetValue(id, out var resource)) + { + return resource.State; + } + return ResourceState.Unloaded; + } + + /// + /// Decrements the reference count of a resource by its Guid. + /// + public static void Release(Guid id) + { + if (_resourceCache.TryGetValue(id, out var resource)) + { + Release(resource); + } + } + + /// + /// Decrements the reference count of a resource. + /// If the count reaches 0, the asset is automatically unloaded from memory. + /// + public static void Release(Resource resource) + { + if (resource == null) return; + + lock (resource) + { + resource.ReferenceCount--; + + if (resource.ReferenceCount <= 0) + { + // Fully unused! We should unload it safely. + if (resource.State == ResourceState.Loaded) + { + resource.Unload(); + } + + resource.State = ResourceState.Unloaded; + _resourceCache.TryRemove(resource.Id, out _); + } + } + } + + /// + /// Background thread loop that pulls from the queue and does the slow file I/O operations. + /// + private static void LoaderWorkerLoop() + { + while (_isRunning) + { + if (_loadQueue.TryDequeue(out var resource)) + { + // Safety check: Was the resource released before we even got around to loading it? + if (resource.ReferenceCount <= 0) + { + resource.State = ResourceState.Unloaded; + continue; + } + + try + { + resource.State = ResourceState.Loading; + string fullPath = Path.Combine(AppContext.BaseDirectory, RootName, resource.Id.ToString()) + ".bin"; + + // Do the slow synchronous disk read + using var stream = StorageContainer.OpenStream(fullPath); + resource.Load(stream); + + resource.State = ResourceState.Loaded; + } + catch (Exception e) + { + Log.Error($"Failed to background load asset '{resource.Path}': {e.Message}"); + resource.State = ResourceState.Failed; + } + } + else + { + // Sleep cleanly if queue is empty to avoid burning total CPU usage on an infinite while-loop + Thread.Sleep(10); + } + } } } diff --git a/Nerfed.Runtime/Resource/SampleMeshVisualComponent.cs b/Nerfed.Runtime/Resource/SampleMeshVisualComponent.cs new file mode 100644 index 0000000..78e1d44 --- /dev/null +++ b/Nerfed.Runtime/Resource/SampleMeshVisualComponent.cs @@ -0,0 +1,19 @@ +namespace Nerfed.Runtime.Resources; + +/// +/// A sample component demonstrating how to use strongly-typed asset references +/// in a realistic scenario where an entity requires multiple distinct resources. +/// +public struct SampleMeshVisualComponent +{ + // The user safely assigns a Mesh GUID in the Editor inspector. + public TypedAssetReference VertexShader; + + // The user safely assigns a Material GUID in the Editor inspector. + public TypedAssetReference FragmentShader; + + public SampleMeshVisualComponent(Guid vertexId, Guid fragId) { + VertexShader = new TypedAssetReference(vertexId); + FragmentShader = new TypedAssetReference(fragId); + } +} diff --git a/Nerfed.Runtime/Resource/SampleRenderSystem.cs b/Nerfed.Runtime/Resource/SampleRenderSystem.cs new file mode 100644 index 0000000..cb18bab --- /dev/null +++ b/Nerfed.Runtime/Resource/SampleRenderSystem.cs @@ -0,0 +1,53 @@ +using MoonTools.ECS; +using Nerfed.Runtime.Scene.Streaming; +using System; + +namespace Nerfed.Runtime.Resources; + +/// +/// A typical rendering preparation system that natively resolves and requests +/// asynchronous background loading for its own required assets, removing the +/// need for a monolithic generic AssetStreaming manager. +/// +public class SampleRenderSystem : MoonTools.ECS.System +{ + private readonly Filter _meshVisualsFilter; + + public SampleRenderSystem(World world) : base(world) { + _meshVisualsFilter = FilterBuilder + .Include() + // Always ignore chunk entities technically "unloading" from RAM + .Exclude() + .Build(); + } + + public override void Update(TimeSpan delta) { + foreach(Entity entity in _meshVisualsFilter.Entities) { + SampleMeshVisualComponent visualComp = Get(entity); + + // 1. Resolve State + ResourceState vertState = ResourceManager.GetState(visualComp.VertexShader.AssetId); + ResourceState fragState = ResourceManager.GetState(visualComp.FragmentShader.AssetId); + + // 2. Asynchronously request assets if they don't exist in memory yet + if(vertState == ResourceState.Unloaded) { + ResourceManager.Retain(visualComp.VertexShader.AssetId, "Unknown/Path"); + } + if(fragState == ResourceState.Unloaded) { + ResourceManager.Retain(visualComp.FragmentShader.AssetId, "Unknown/Path"); + } + + // 3. Prevent rendering logic unless ALL strictly required assets are fully mapped + bool isReadyToDraw = vertState == ResourceState.Loaded && fragState == ResourceState.Loaded; + + if(isReadyToDraw) { + // At this exact point, you can safely assume: + // 1) The background loading threads are 100% finished processing these shaders. + // 2) The GraphicsDevice can safely extract the native handle. + + // e.g. GraphicsDevice.BindShader(visualComp.VertexShader.AssetId); + // e.g. GraphicsDevice.DrawPolygons(...); + } + } + } +}