Allocate one large buffer for a heap and offset into it.va-merge

Greatly reduce VA allocations we have to make and makes returned VA more
sensible. D3D12 usage flags for buffers seem generic enough that there
is no obvious benefit to place smaller VkBuffers on top of
VkDeviceMemory.

4 files changed, 163 insertions, 24 deletions

diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
index d420863b..545ef642 100644
--- a/libs/vkd3d/command.c
+++ b/libs/vkd3d/command.c
@@ -4076,8 +4076,17 @@ static void d3d12_command_list_set_root_cbv(struct d3d12_command_list *list,
 
     resource = vkd3d_gpu_va_allocator_dereference(&list->device->gpu_va_allocator, gpu_address);
     buffer_info.buffer = resource->u.vk_buffer;
-    buffer_info.offset = gpu_address - resource->gpu_address;
-    buffer_info.range = resource->desc.Width - buffer_info.offset;
+
+    if (resource->placed_buffer)
+    {
+        buffer_info.offset = gpu_address - resource->gpu_address + resource->heap_offset;
+        buffer_info.range = resource->heap->buffer_resource->desc.Width - buffer_info.offset;
+    }
+    else
+    {
+        buffer_info.offset = gpu_address - resource->gpu_address;
+        buffer_info.range = resource->desc.Width - buffer_info.offset;
+    }
     buffer_info.range = min(buffer_info.range, vk_info->device_limits.maxUniformBufferRange);
 
     if (vk_info->KHR_push_descriptor)
@@ -4257,7 +4266,7 @@ static void STDMETHODCALLTYPE d3d12_command_list_IASetIndexBuffer(ID3D12Graphics
 
     resource = vkd3d_gpu_va_allocator_dereference(&list->device->gpu_va_allocator, view->BufferLocation);
     VK_CALL(vkCmdBindIndexBuffer(list->vk_command_buffer, resource->u.vk_buffer,
-            view->BufferLocation - resource->gpu_address, index_type));
+            view->BufferLocation - resource->gpu_address + resource->heap_offset, index_type));
 }
 
 static void STDMETHODCALLTYPE d3d12_command_list_IASetVertexBuffers(ID3D12GraphicsCommandList1 *iface,
@@ -4291,7 +4300,7 @@ static void STDMETHODCALLTYPE d3d12_command_list_IASetVertexBuffers(ID3D12Graphi
         {
             resource = vkd3d_gpu_va_allocator_dereference(gpu_va_allocator, views[i].BufferLocation);
             buffers[i] = resource->u.vk_buffer;
-            offsets[i] = views[i].BufferLocation - resource->gpu_address;
+            offsets[i] = views[i].BufferLocation - resource->gpu_address + resource->heap_offset;
             stride = views[i].StrideInBytes;
         }
         else
@@ -4351,12 +4360,13 @@ static void STDMETHODCALLTYPE d3d12_command_list_SOSetTargets(ID3D12GraphicsComm
         {
             resource = vkd3d_gpu_va_allocator_dereference(gpu_va_allocator, views[i].BufferLocation);
             buffers[count] = resource->u.vk_buffer;
-            offsets[count] = views[i].BufferLocation - resource->gpu_address;
+            offsets[count] = views[i].BufferLocation - resource->gpu_address + resource->heap_offset;
             sizes[count] = views[i].SizeInBytes;
 
             resource = vkd3d_gpu_va_allocator_dereference(gpu_va_allocator, views[i].BufferFilledSizeLocation);
             list->so_counter_buffers[start_slot + i] = resource->u.vk_buffer;
-            list->so_counter_buffer_offsets[start_slot + i] = views[i].BufferFilledSizeLocation - resource->gpu_address;
+            list->so_counter_buffer_offsets[start_slot + i] =
+                    views[i].BufferFilledSizeLocation - resource->gpu_address + resource->heap_offset;
             ++count;
         }
         else
diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c
index 274657d8..46d6b7df 100644
--- a/libs/vkd3d/device.c
+++ b/libs/vkd3d/device.c
@@ -1335,7 +1335,9 @@ static HRESULT vkd3d_init_device_caps(struct d3d12_device *device,
     device->feature_options.CrossAdapterRowMajorTextureSupported = FALSE;
     /* SPV_EXT_shader_viewport_index_layer */
     device->feature_options.VPAndRTArrayIndexFromAnyShaderFeedingRasterizerSupportedWithoutGSEmulation = FALSE;
-    device->feature_options.ResourceHeapTier = D3D12_RESOURCE_HEAP_TIER_2;
+
+    /* Need to make sure buffers are contained in their own heaps. */
+    device->feature_options.ResourceHeapTier = D3D12_RESOURCE_HEAP_TIER_1;
 
     if ((vr = VK_CALL(vkEnumerateDeviceExtensionProperties(physical_device, NULL, &count, NULL))) < 0)
     {
@@ -1909,6 +1911,7 @@ static D3D12_GPU_VIRTUAL_ADDRESS vkd3d_gpu_va_allocator_allocate_slab(struct vkd
                     (unsigned long long)virtual_address, vacant_index, vkd3d_va_size_class_tag[size_class]);
             assert(!allocator->slab_mem_allocations[size_class][vacant_index].ptr);
             allocator->slab_mem_allocations[size_class][vacant_index].ptr = ptr;
+            allocator->slab_mem_allocations[size_class][vacant_index].size = size;
         }
 
         /* Try higher size class. Very large allocations should be rare enough that we can use the higher size areas
@@ -1991,6 +1994,12 @@ static void *vkd3d_gpu_va_allocator_dereference_slab(struct vkd3d_gpu_va_allocat
     }
 
     slab = &allocator->slab_mem_allocations[size_class][base_index];
+    base_offset -= base_index * vkd3d_va_size_class_size[size_class];
+    if (base_offset >= slab->size)
+    {
+        ERR("Accessed slab out of range.\n");
+        return NULL;
+    }
     return slab->ptr;
 }
 
diff --git a/libs/vkd3d/resource.c b/libs/vkd3d/resource.c
index 4604efd3..7ade4bac 100644
--- a/libs/vkd3d/resource.c
+++ b/libs/vkd3d/resource.c
@@ -292,6 +292,8 @@ static ULONG STDMETHODCALLTYPE d3d12_heap_AddRef(ID3D12Heap *iface)
     return refcount;
 }
 
+static ULONG d3d12_resource_decref(struct d3d12_resource *resource);
+
 static void d3d12_heap_destroy(struct d3d12_heap *heap)
 {
     struct d3d12_device *device = heap->device;
@@ -299,6 +301,9 @@ static void d3d12_heap_destroy(struct d3d12_heap *heap)
 
     TRACE("Destroying heap %p.\n", heap);
 
+    if (heap->buffer_resource)
+        d3d12_resource_decref(heap->buffer_resource);
+
     if (heap->is_persistent && heap->map_ptr)
         VK_CALL(vkUnmapMemory(device->vk_device, heap->vk_memory));
 
@@ -542,6 +547,19 @@ done:
 
 static HRESULT validate_heap_desc(const D3D12_HEAP_DESC *desc, const struct d3d12_resource *resource)
 {
+    unsigned deny_count = 0;
+    if (desc->Flags & D3D12_HEAP_FLAG_DENY_BUFFERS)
+        deny_count++;
+    if (desc->Flags & D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES)
+        deny_count++;
+    if (desc->Flags & D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES)
+        deny_count++;
+
+    if (deny_count < 2)
+    {
+        WARN("(flag = 0x%x) Heap tier 1 does not support mixing resource types in same heap.\n", desc->Flags);
+    }
+
     if (!resource && !desc->SizeInBytes)
     {
         WARN("Invalid size %"PRIu64".\n", desc->SizeInBytes);
@@ -564,6 +582,12 @@ static HRESULT validate_heap_desc(const D3D12_HEAP_DESC *desc, const struct d3d1
     return S_OK;
 }
 
+static HRESULT d3d12_resource_create(struct d3d12_device *device,
+                                     const D3D12_HEAP_PROPERTIES *heap_properties, D3D12_HEAP_FLAGS heap_flags,
+                                     const D3D12_RESOURCE_DESC *desc, D3D12_RESOURCE_STATES initial_state,
+                                     const D3D12_CLEAR_VALUE *optimized_clear_value, bool placed,
+                                     struct d3d12_resource **resource);
+
 static HRESULT d3d12_heap_init(struct d3d12_heap *heap,
         struct d3d12_device *device, const D3D12_HEAP_DESC *desc, const struct d3d12_resource *resource)
 {
@@ -571,6 +595,9 @@ static HRESULT d3d12_heap_init(struct d3d12_heap *heap,
     VkDeviceSize vk_memory_size;
     HRESULT hr;
     int rc;
+    bool buffers_allowed;
+    D3D12_RESOURCE_DESC resource_desc;
+    D3D12_RESOURCE_STATES initial_resource_state;
 
     heap->ID3D12Heap_iface.lpVtbl = &d3d12_heap_vtbl;
     heap->refcount = 1;
@@ -582,6 +609,7 @@ static HRESULT d3d12_heap_init(struct d3d12_heap *heap,
 
     heap->map_ptr = NULL;
     heap->map_count = 0;
+    heap->buffer_resource = NULL;
 
     if (!heap->desc.Properties.CreationNodeMask)
         heap->desc.Properties.CreationNodeMask = 1;
@@ -609,6 +637,53 @@ static HRESULT d3d12_heap_init(struct d3d12_heap *heap,
         return hr;
     }
 
+    buffers_allowed = !(heap->desc.Flags & D3D12_HEAP_FLAG_DENY_BUFFERS);
+    if (buffers_allowed && !resource)
+    {
+        /* Create a single omnipotent buffer which fills the entire heap.
+         * Whenever we place buffer resources on this heap, we'll just offset this VkBuffer.
+         * This allows us to keep VA space somewhat sane.
+         * One possible downside is that the buffer might be slightly slower to access. */
+
+        memset(&resource_desc, 0, sizeof(resource_desc));
+        resource_desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
+        resource_desc.Width = desc->SizeInBytes;
+        resource_desc.Height = 1;
+        resource_desc.DepthOrArraySize = 1;
+        resource_desc.MipLevels = 1;
+        resource_desc.SampleDesc.Count = 1;
+        resource_desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+
+        switch (desc->Properties.Type)
+        {
+        case D3D12_HEAP_TYPE_DEFAULT:
+            /* Upload and readback heaps do not allow UAV access, only enable this flag for GPU heap. */
+            resource_desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
+            initial_resource_state = D3D12_RESOURCE_STATE_COMMON;
+            break;
+
+        case D3D12_HEAP_TYPE_UPLOAD:
+            initial_resource_state = D3D12_RESOURCE_STATE_GENERIC_READ;
+            break;
+
+        case D3D12_HEAP_TYPE_READBACK:
+            initial_resource_state = D3D12_RESOURCE_STATE_COPY_DEST;
+            break;
+
+        default:
+            ERR("Unknown heap.\n");
+            return E_INVALIDARG;
+        }
+
+        if (FAILED(hr = d3d12_resource_create(device, &desc->Properties, desc->Flags,
+                                              &resource_desc, initial_resource_state,
+                                              NULL, false, &heap->buffer_resource)))
+        {
+            heap->buffer_resource = NULL;
+            return hr;
+        }
+    }
+
     if (resource)
     {
         if (d3d12_resource_is_buffer(resource))
@@ -626,12 +701,19 @@ static HRESULT d3d12_heap_init(struct d3d12_heap *heap,
 
         heap->desc.SizeInBytes = vk_memory_size;
     }
+    else if (heap->buffer_resource)
+    {
+        hr = vkd3d_allocate_buffer_memory(device, heap->buffer_resource->u.vk_buffer,
+                                          &heap->desc.Properties, heap->desc.Flags,
+                                          &heap->vk_memory, &heap->vk_memory_type, &vk_memory_size);
+    }
     else
     {
+        /* Allocate generic memory which should hopefully match up with whatever resources
+         * we want to place here. */
         memory_requirements.size = heap->desc.SizeInBytes;
         memory_requirements.alignment = heap->desc.Alignment;
         memory_requirements.memoryTypeBits = ~(uint32_t)0;
-
         hr = vkd3d_allocate_device_memory(device, &heap->desc.Properties,
                 heap->desc.Flags, &memory_requirements, NULL,
                 &heap->vk_memory, &heap->vk_memory_type);
@@ -1046,13 +1128,16 @@ static void d3d12_resource_destroy(struct d3d12_resource *resource, struct d3d12
     if (resource->flags & VKD3D_RESOURCE_EXTERNAL)
         return;
 
-    if (resource->gpu_address)
-        vkd3d_gpu_va_allocator_free(&device->gpu_va_allocator, resource->gpu_address);
+    if (!resource->placed_buffer)
+    {
+        if (resource->gpu_address)
+            vkd3d_gpu_va_allocator_free(&device->gpu_va_allocator, resource->gpu_address);
 
-    if (d3d12_resource_is_buffer(resource))
-        VK_CALL(vkDestroyBuffer(device->vk_device, resource->u.vk_buffer, NULL));
-    else
-        VK_CALL(vkDestroyImage(device->vk_device, resource->u.vk_image, NULL));
+        if (d3d12_resource_is_buffer(resource))
+            VK_CALL(vkDestroyBuffer(device->vk_device, resource->u.vk_buffer, NULL));
+        else
+            VK_CALL(vkDestroyImage(device->vk_device, resource->u.vk_image, NULL));
+    }
 
     if (resource->flags & VKD3D_RESOURCE_DEDICATED_HEAP)
         d3d12_heap_destroy(resource->heap);
@@ -1613,7 +1698,7 @@ static bool d3d12_resource_validate_heap_properties(const struct d3d12_resource
 static HRESULT d3d12_resource_init(struct d3d12_resource *resource, struct d3d12_device *device,
         const D3D12_HEAP_PROPERTIES *heap_properties, D3D12_HEAP_FLAGS heap_flags,
         const D3D12_RESOURCE_DESC *desc, D3D12_RESOURCE_STATES initial_state,
-        const D3D12_CLEAR_VALUE *optimized_clear_value)
+        const D3D12_CLEAR_VALUE *optimized_clear_value, bool placed)
 {
     HRESULT hr;
 
@@ -1643,6 +1728,7 @@ static HRESULT d3d12_resource_init(struct d3d12_resource *resource, struct d3d12
 
     resource->gpu_address = 0;
     resource->flags = 0;
+    resource->placed_buffer = placed;
 
     if (FAILED(hr = d3d12_resource_validate_desc(&resource->desc)))
         return hr;
@@ -1650,6 +1736,13 @@ static HRESULT d3d12_resource_init(struct d3d12_resource *resource, struct d3d12
     switch (desc->Dimension)
     {
         case D3D12_RESOURCE_DIMENSION_BUFFER:
+            /* We'll inherit a VkBuffer reference from the heap with an implied offset. */
+            if (placed)
+            {
+                resource->u.vk_buffer = VK_NULL_HANDLE;
+                break;
+            }
+
             if (FAILED(hr = vkd3d_create_buffer(device, heap_properties, heap_flags,
                     &resource->desc, &resource->u.vk_buffer)))
                 return hr;
@@ -1699,7 +1792,7 @@ static HRESULT d3d12_resource_init(struct d3d12_resource *resource, struct d3d12
 static HRESULT d3d12_resource_create(struct d3d12_device *device,
         const D3D12_HEAP_PROPERTIES *heap_properties, D3D12_HEAP_FLAGS heap_flags,
         const D3D12_RESOURCE_DESC *desc, D3D12_RESOURCE_STATES initial_state,
-        const D3D12_CLEAR_VALUE *optimized_clear_value, struct d3d12_resource **resource)
+        const D3D12_CLEAR_VALUE *optimized_clear_value, bool placed, struct d3d12_resource **resource)
 {
     struct d3d12_resource *object;
     HRESULT hr;
@@ -1708,7 +1801,7 @@ static HRESULT d3d12_resource_create(struct d3d12_device *device,
         return E_OUTOFMEMORY;
 
     if (FAILED(hr = d3d12_resource_init(object, device, heap_properties, heap_flags,
-            desc, initial_state, optimized_clear_value)))
+            desc, initial_state, optimized_clear_value, placed)))
     {
         vkd3d_free(object);
         return hr;
@@ -1750,7 +1843,7 @@ HRESULT d3d12_committed_resource_create(struct d3d12_device *device,
     }
 
     if (FAILED(hr = d3d12_resource_create(device, heap_properties, heap_flags,
-            desc, initial_state, optimized_clear_value, &object)))
+            desc, initial_state, optimized_clear_value, &object, false)))
         return hr;
 
     if (FAILED(hr = vkd3d_allocate_resource_memory(device, object, heap_properties, heap_flags)))
@@ -1774,6 +1867,16 @@ static HRESULT vkd3d_bind_heap_memory(struct d3d12_device *device,
     VkMemoryRequirements requirements;
     VkResult vr;
 
+    if (resource->placed_buffer)
+    {
+        /* Just inherit the buffer from the heap. */
+        resource->u.vk_buffer = heap->buffer_resource->u.vk_buffer;
+        resource->heap = heap;
+        resource->heap_offset = heap_offset;
+        resource->gpu_address = heap->buffer_resource->gpu_address + heap_offset;
+        return S_OK;
+    }
+
     if (d3d12_resource_is_buffer(resource))
         VK_CALL(vkGetBufferMemoryRequirements(vk_device, resource->u.vk_buffer, &requirements));
     else
@@ -1823,7 +1926,7 @@ HRESULT d3d12_placed_resource_create(struct d3d12_device *device, struct d3d12_h
     HRESULT hr;
 
     if (FAILED(hr = d3d12_resource_create(device, &heap->desc.Properties, heap->desc.Flags,
-            desc, initial_state, optimized_clear_value, &object)))
+            desc, initial_state, optimized_clear_value, true, &object)))
         return hr;
 
     if (FAILED(hr = vkd3d_bind_heap_memory(device, object, heap, heap_offset)))
@@ -1847,7 +1950,7 @@ HRESULT d3d12_reserved_resource_create(struct d3d12_device *device,
     HRESULT hr;
 
     if (FAILED(hr = d3d12_resource_create(device, NULL, 0,
-            desc, initial_state, optimized_clear_value, &object)))
+            desc, initial_state, optimized_clear_value, false, &object)))
         return hr;
 
     TRACE("Created reserved resource %p.\n", object);
@@ -2149,7 +2252,7 @@ static bool vkd3d_create_buffer_view_for_resource(struct d3d12_device *device,
     assert(d3d12_resource_is_buffer(resource));
 
     return vkd3d_create_buffer_view(device, resource->u.vk_buffer,
-            format, offset * element_size, size * element_size, view);
+            format, resource->heap_offset + offset * element_size, size * element_size, view);
 }
 
 static void vkd3d_set_view_swizzle_for_format(VkComponentMapping *components,
@@ -2415,7 +2518,7 @@ void d3d12_desc_create_cbv(struct d3d12_desc *descriptor,
     {
         resource = vkd3d_gpu_va_allocator_dereference(&device->gpu_va_allocator, desc->BufferLocation);
         buffer_info->buffer = resource->u.vk_buffer;
-        buffer_info->offset = desc->BufferLocation - resource->gpu_address;
+        buffer_info->offset = desc->BufferLocation - resource->gpu_address + resource->heap_offset;
         buffer_info->range = min(desc->SizeInBytes, resource->desc.Width - buffer_info->offset);
     }
     else
@@ -2751,7 +2854,7 @@ static void vkd3d_create_buffer_uav(struct d3d12_desc *descriptor, struct d3d12_
 
         format = vkd3d_get_format(device, DXGI_FORMAT_R32_UINT, false);
         if (!vkd3d_create_vk_buffer_view(device, counter_resource->u.vk_buffer, format,
-                desc->u.Buffer.CounterOffsetInBytes, sizeof(uint32_t), &view->vk_counter_view))
+                desc->u.Buffer.CounterOffsetInBytes + resource->heap_offset, sizeof(uint32_t), &view->vk_counter_view))
         {
             WARN("Failed to create counter buffer view.\n");
             view->vk_counter_view = VK_NULL_HANDLE;
@@ -2857,12 +2960,26 @@ bool vkd3d_create_raw_buffer_view(struct d3d12_device *device,
 {
     const struct vkd3d_format *format;
     struct d3d12_resource *resource;
+    uint64_t range;
+    uint64_t offset;
 
     format = vkd3d_get_format(device, DXGI_FORMAT_R32_UINT, false);
     resource = vkd3d_gpu_va_allocator_dereference(&device->gpu_va_allocator, gpu_address);
     assert(d3d12_resource_is_buffer(resource));
+
+    if (resource->placed_buffer)
+    {
+        offset = gpu_address - resource->gpu_address + resource->heap_offset;
+        range = min(resource->heap->buffer_resource->desc.Width - offset, device->vk_info.device_limits.maxStorageBufferRange);
+    }
+    else
+    {
+        offset = gpu_address - resource->gpu_address;
+        range = VK_WHOLE_SIZE;
+    }
+
     return vkd3d_create_vk_buffer_view(device, resource->u.vk_buffer, format,
-            gpu_address - resource->gpu_address, VK_WHOLE_SIZE, vk_buffer_view);
+            gpu_address - resource->gpu_address + resource->heap_offset, range, vk_buffer_view);
 }
 
 /* samplers */
diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
index 92376b73..56b31cac 100644
--- a/libs/vkd3d/vkd3d_private.h
+++ b/libs/vkd3d/vkd3d_private.h
@@ -243,6 +243,7 @@ struct vkd3d_gpu_va_allocation
 struct vkd3d_gpu_va_slab_entry
 {
     void *ptr;
+    SIZE_T size;
 };
 
 struct vkd3d_gpu_va_allocator
@@ -411,6 +412,7 @@ struct d3d12_heap
     unsigned int map_count;
     uint32_t vk_memory_type;
 
+    struct d3d12_resource *buffer_resource;
     struct d3d12_device *device;
 
     struct vkd3d_private_store private_store;
@@ -447,6 +449,7 @@ struct d3d12_resource
 
     struct d3d12_heap *heap;
     uint64_t heap_offset;
+    bool placed_buffer;
 
     D3D12_RESOURCE_STATES initial_state;
     D3D12_RESOURCE_STATES present_state;