kern: implement kmemoryblock/kmemoryinfo merge disable

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -176,15 +176,31 @@ namespace ams::kern {
         KMemoryAttribute_SetMask        = KMemoryAttribute_Uncached,
     };
 
+    enum KMemoryBlockDisableMergeAttribute : u8 {
+        KMemoryBlockDisableMergeAttribute_None        = 0,
+        KMemoryBlockDisableMergeAttribute_Normal      = (1u << 0),
+        KMemoryBlockDisableMergeAttribute_DeviceLeft  = (1u << 1),
+        KMemoryBlockDisableMergeAttribute_IpcLeft     = (1u << 2),
+        KMemoryBlockDisableMergeAttribute_Locked      = (1u << 3),
+        KMemoryBlockDisableMergeAttribute_DeviceRight = (1u << 4),
+
+        KMemoryBlockDisableMergeAttribute_AllLeft  = KMemoryBlockDisableMergeAttribute_Normal | KMemoryBlockDisableMergeAttribute_DeviceLeft | KMemoryBlockDisableMergeAttribute_IpcLeft | KMemoryBlockDisableMergeAttribute_Locked,
+        KMemoryBlockDisableMergeAttribute_AllRight = KMemoryBlockDisableMergeAttribute_DeviceRight,
+    };
+
     struct KMemoryInfo {
         uintptr_t address;
         size_t size;
         KMemoryState state;
+        u16 device_disable_merge_left_count;
+        u16 device_disable_merge_right_count;
+        u16 ipc_lock_count;
+        u16 device_use_count;
+        u16 ipc_disable_merge_count;
         KMemoryPermission perm;
         KMemoryAttribute  attribute;
         KMemoryPermission original_perm;
-        u16 ipc_lock_count;
-        u16 device_use_count;
+        KMemoryBlockDisableMergeAttribute disable_merge_attribute;
 
         constexpr ams::svc::MemoryInfo GetSvcMemoryInfo() const {
             return {
@@ -223,6 +239,10 @@ namespace ams::kern {
             return this->ipc_lock_count;
         }
 
+        constexpr u16 GetIpcDisableMergeCount() const {
+            return this->ipc_disable_merge_count;
+        }
+
         constexpr KMemoryState GetState() const {
             return this->state;
         }
@@ -238,18 +258,26 @@ namespace ams::kern {
         constexpr KMemoryAttribute GetAttribute() const {
             return this->attribute;
         }
+
+        constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const {
+            return this->disable_merge_attribute;
+        }
     };
 
     class KMemoryBlock : public util::IntrusiveRedBlackTreeBaseNode<KMemoryBlock> {
         private:
+            u16 device_disable_merge_left_count;
+            u16 device_disable_merge_right_count;
             KProcessAddress address;
             size_t num_pages;
             KMemoryState memory_state;
             u16 ipc_lock_count;
             u16 device_use_count;
+            u16 ipc_disable_merge_count;
             KMemoryPermission perm;
             KMemoryPermission original_perm;
             KMemoryAttribute attribute;
+            KMemoryBlockDisableMergeAttribute disable_merge_attribute;
         public:
             static constexpr ALWAYS_INLINE int Compare(const KMemoryBlock &lhs, const KMemoryBlock &rhs) {
                 if (lhs.GetAddress() < rhs.GetAddress()) {
@@ -285,6 +313,10 @@ namespace ams::kern {
                 return this->ipc_lock_count;
             }
 
+            constexpr u16 GetIpcDisableMergeCount() const {
+                return this->ipc_disable_merge_count;
+            }
+
             constexpr KMemoryPermission GetPermission() const {
                 return this->perm;
             }
@@ -299,25 +331,29 @@ namespace ams::kern {
 
             constexpr KMemoryInfo GetMemoryInfo() const {
                 return {
-                    .address          = GetInteger(this->GetAddress()),
-                    .size             = this->GetSize(),
-                    .state            = this->memory_state,
-                    .perm             = this->perm,
-                    .attribute        = this->attribute,
-                    .original_perm    = this->original_perm,
-                    .ipc_lock_count   = this->ipc_lock_count,
-                    .device_use_count = this->device_use_count,
+                    .address                          = GetInteger(this->GetAddress()),
+                    .size                             = this->GetSize(),
+                    .state                            = this->memory_state,
+                    .device_disable_merge_left_count  = this->device_disable_merge_left_count,
+                    .device_disable_merge_right_count = this->device_disable_merge_right_count,
+                    .ipc_lock_count                   = this->ipc_lock_count,
+                    .device_use_count                 = this->device_use_count,
+                    .ipc_disable_merge_count          = this->ipc_disable_merge_count,
+                    .perm                             = this->perm,
+                    .attribute                        = this->attribute,
+                    .original_perm                    = this->original_perm,
+                    .disable_merge_attribute          = this->disable_merge_attribute,
                 };
             }
         public:
             constexpr KMemoryBlock()
-                : address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), perm(), original_perm(), attribute()
+                : device_disable_merge_left_count(), device_disable_merge_right_count(), address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), ipc_disable_merge_count(), perm(), original_perm(), attribute(), disable_merge_attribute()
             {
                 /* ... */
             }
 
             constexpr KMemoryBlock(KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr)
-                : address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), perm(p), original_perm(KMemoryPermission_None), attribute(attr)
+                : device_disable_merge_left_count(), device_disable_merge_right_count(), address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), perm(p), ipc_disable_merge_count(), original_perm(KMemoryPermission_None), attribute(attr), disable_merge_attribute()
             {
                 /* ... */
             }
@@ -351,21 +387,29 @@ namespace ams::kern {
                        this->device_use_count == rhs.device_use_count;
             }
 
+            constexpr bool CanMergeWith(const KMemoryBlock &rhs) const {
+                return this->HasSameProperties(rhs) &&
+                      (this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight) == 0 &&
+                      (rhs.disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft) == 0;
+            }
+
             constexpr bool Contains(KProcessAddress addr) const {
                 MESOSPHERE_ASSERT_THIS();
 
                 return this->GetAddress() <= addr && addr <= this->GetEndAddress();
             }
 
-            constexpr void Add(size_t np) {
+            constexpr void Add(const KMemoryBlock &added_block) {
                 MESOSPHERE_ASSERT_THIS();
-                MESOSPHERE_ASSERT(np > 0);
-                MESOSPHERE_ASSERT(this->GetAddress() + np * PageSize - 1 < this->GetEndAddress() + np * PageSize - 1);
+                MESOSPHERE_ASSERT(added_block.GetNumPages() > 0);
+                MESOSPHERE_ASSERT(this->GetAddress() + added_block.GetSize() - 1 < this->GetEndAddress() + added_block.GetSize() - 1);
 
-                this->num_pages += np;
+                this->num_pages += added_block.GetNumPages();
+                this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | added_block.disable_merge_attribute);
+                this->device_disable_merge_right_count = added_block.device_disable_merge_right_count;
             }
 
-            constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) {
+            constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a, bool set_disable_merge_attr, u8 set_mask, u8 clear_mask) {
                 MESOSPHERE_ASSERT_THIS();
                 MESOSPHERE_ASSERT(this->original_perm == KMemoryPermission_None);
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == 0);
@@ -373,6 +417,13 @@ namespace ams::kern {
                 this->memory_state = s;
                 this->perm         = p;
                 this->attribute    = static_cast<KMemoryAttribute>(a | (this->attribute & (KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared)));
+
+                if (set_disable_merge_attr && set_mask != 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | set_mask);
+                }
+                if (clear_mask != 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~clear_mask);
+                }
             }
 
             constexpr void Split(KMemoryBlock *block, KProcessAddress addr) {
@@ -381,20 +432,55 @@ namespace ams::kern {
                 MESOSPHERE_ASSERT(this->Contains(addr));
                 MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize));
 
-                block->address          = this->address;
-                block->num_pages        = (addr - this->GetAddress()) / PageSize;
-                block->memory_state     = this->memory_state;
-                block->ipc_lock_count   = this->ipc_lock_count;
-                block->device_use_count = this->device_use_count;
-                block->perm             = this->perm;
-                block->original_perm    = this->original_perm;
-                block->attribute        = this->attribute;
+                block->address                          = this->address;
+                block->num_pages                        = (addr - this->GetAddress()) / PageSize;
+                block->memory_state                     = this->memory_state;
+                block->ipc_lock_count                   = this->ipc_lock_count;
+                block->device_use_count                 = this->device_use_count;
+                block->perm                             = this->perm;
+                block->original_perm                    = this->original_perm;
+                block->attribute                        = this->attribute;
+                block->disable_merge_attribute          = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft);
+                block->ipc_disable_merge_count          = this->ipc_disable_merge_count;
+                block->device_disable_merge_left_count  = this->device_disable_merge_left_count;
+                block->device_disable_merge_right_count = 0;
 
                 this->address = addr;
                 this->num_pages -= block->num_pages;
+
+                this->ipc_disable_merge_count         = 0;
+                this->device_disable_merge_left_count = 0;
+                this->disable_merge_attribute         = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight);
             }
 
-            constexpr void ShareToDevice(KMemoryPermission new_perm) {
+            constexpr void UpdateDeviceDisableMergeStateForShareLeft(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/right aren't used. */
+                MESOSPHERE_UNUSED(new_perm, right);
+
+                if (left) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceLeft);
+                    const u16 new_device_disable_merge_left_count = ++this->device_disable_merge_left_count;
+                    MESOSPHERE_ABORT_UNLESS(device_disable_merge_left_count > 0);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForShareRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/left aren't used. */
+                MESOSPHERE_UNUSED(new_perm, left);
+
+                if (right) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceRight);
+                    const u16 new_device_disable_merge_right_count = ++this->device_disable_merge_right_count;
+                    MESOSPHERE_ABORT_UNLESS(new_device_disable_merge_right_count > 0);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForShare(KMemoryPermission new_perm, bool left, bool right) {
+                this->UpdateDeviceDisableMergeStateForShareLeft(new_perm, left, right);
+                this->UpdateDeviceDisableMergeStateForShareRight(new_perm, left, right);
+            }
+
+            constexpr void ShareToDevice(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -406,9 +492,47 @@ namespace ams::kern {
                 MESOSPHERE_ABORT_UNLESS(new_count > 0);
 
                 this->attribute = static_cast<KMemoryAttribute>(this->attribute | KMemoryAttribute_DeviceShared);
+
+                this->UpdateDeviceDisableMergeStateForShare(new_perm, left, right);
             }
 
-            constexpr void UnshareToDevice(KMemoryPermission new_perm) {
+            constexpr void UpdateDeviceDisableMergeStateForUnshareLeft(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/right aren't used. */
+                MESOSPHERE_UNUSED(new_perm, right);
+
+                if (left) {
+                    if (!this->device_disable_merge_left_count) {
+                        return;
+                    }
+                    --this->device_disable_merge_left_count;
+                }
+
+                this->device_disable_merge_left_count = std::min(this->device_disable_merge_left_count, this->device_use_count);
+
+                if (this->device_disable_merge_left_count == 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceLeft);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForUnshareRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/left aren't used. */
+                MESOSPHERE_UNUSED(new_perm, left);
+
+                if (right) {
+                    const u16 old_device_disable_merge_right_count = this->device_disable_merge_right_count--;
+                    MESOSPHERE_ASSERT(old_device_disable_merge_right_count > 0);
+                    if (old_device_disable_merge_right_count == 1) {
+                        this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceRight);
+                    }
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForUnshare(KMemoryPermission new_perm, bool left, bool right) {
+                this->UpdateDeviceDisableMergeStateForUnshareLeft(new_perm, left, right);
+                this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right);
+            }
+
+            constexpr void UnshareToDevice(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -422,9 +546,29 @@ namespace ams::kern {
                 if (old_count == 1) {
                     this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_DeviceShared);
                 }
+
+                this->UpdateDeviceDisableMergeStateForUnshare(new_perm, left, right);
             }
 
-            constexpr void LockForIpc(KMemoryPermission new_perm) {
+            constexpr void UnshareToDeviceRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission isn't used. */
+                MESOSPHERE_UNUSED(new_perm);
+
+                /* We must be shared. */
+                MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_DeviceShared) == KMemoryAttribute_DeviceShared);
+
+                /* Unhare. */
+                const u16 old_count = this->device_use_count--;
+                MESOSPHERE_ABORT_UNLESS(old_count > 0);
+
+                if (old_count == 1) {
+                    this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_DeviceShared);
+                }
+
+                this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right);
+            }
+
+            constexpr void LockForIpc(KMemoryPermission new_perm, bool left, bool right) {
                 /* We must either be locked or have a zero lock count. */
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == KMemoryAttribute_IpcLocked || this->ipc_lock_count == 0);
 
@@ -441,9 +585,16 @@ namespace ams::kern {
                     this->perm          = static_cast<KMemoryPermission>((new_perm & KMemoryPermission_IpcLockChangeMask) | (this->original_perm & ~KMemoryPermission_IpcLockChangeMask));
                 }
                 this->attribute = static_cast<KMemoryAttribute>(this->attribute | KMemoryAttribute_IpcLocked);
+
+                if (left) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_IpcLeft);
+                    const u16 new_ipc_disable_merge_count = ++this->ipc_disable_merge_count;
+                    MESOSPHERE_ABORT_UNLESS(new_ipc_disable_merge_count > 0);
+                }
+                MESOSPHERE_UNUSED(right);
             }
 
-            constexpr void UnlockForIpc(KMemoryPermission new_perm) {
+            constexpr void UnlockForIpc(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -461,6 +612,19 @@ namespace ams::kern {
                     this->original_perm = KMemoryPermission_None;
                     this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_IpcLocked);
                 }
+
+                if (left) {
+                    const u16 old_ipc_disable_merge_count = this->ipc_disable_merge_count--;
+                    MESOSPHERE_ASSERT(old_ipc_disable_merge_count > 0);
+                    if (old_ipc_disable_merge_count == 1) {
+                        this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_IpcLeft);
+                    }
+                }
+                MESOSPHERE_UNUSED(right);
+            }
+
+            constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const {
+                return this->disable_merge_attribute;
             }
     };
     static_assert(std::is_trivially_destructible<KMemoryBlock>::value);

libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp

@@ -82,6 +82,8 @@ namespace ams::kern {
             MemoryBlockTree memory_block_tree;
             KProcessAddress start_address;
             KProcessAddress end_address;
+        private:
+            void CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages);
         public:
             constexpr KMemoryBlockManager() : memory_block_tree(), start_address(), end_address() { /* ... */ }
 
@@ -94,8 +96,8 @@ namespace ams::kern {
 
             KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const;
 
-            void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr);
-            void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm), KMemoryPermission perm);
+            void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr);
+            void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm, bool left, bool right), KMemoryPermission perm);
 
             void UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr);