using ChocolArm64.Memory; using Ryujinx.Common; using System; using System.Collections.Generic; namespace Ryujinx.HLE.HOS.Kernel { class KMemoryManager { public const int PageSize = 0x1000; private const int KMemoryBlockSize = 0x40; //We need 2 blocks for the case where a big block //needs to be split in 2, plus one block that will be the new one inserted. private const int MaxBlocksNeededForInsertion = 2; private LinkedList _blocks; private MemoryManager _cpuMemory; private Horizon _system; public ulong AddrSpaceStart { get; private set; } public ulong AddrSpaceEnd { get; private set; } public ulong CodeRegionStart { get; private set; } public ulong CodeRegionEnd { get; private set; } public ulong HeapRegionStart { get; private set; } public ulong HeapRegionEnd { get; private set; } private ulong _currentHeapAddr; public ulong AliasRegionStart { get; private set; } public ulong AliasRegionEnd { get; private set; } public ulong StackRegionStart { get; private set; } public ulong StackRegionEnd { get; private set; } public ulong TlsIoRegionStart { get; private set; } public ulong TlsIoRegionEnd { get; private set; } private ulong _heapCapacity; public ulong PhysicalMemoryUsage { get; private set; } private MemoryRegion _memRegion; private bool _aslrDisabled; public int AddrSpaceWidth { get; private set; } private bool _isKernel; private bool _aslrEnabled; private KMemoryBlockAllocator _blockAllocator; private int _contextId; private MersenneTwister _randomNumberGenerator; public KMemoryManager(Horizon system, MemoryManager cpuMemory) { _system = system; _cpuMemory = cpuMemory; _blocks = new LinkedList(); } private static readonly int[] AddrSpaceSizes = new int[] { 32, 36, 32, 39 }; public KernelResult InitializeForProcess( AddressSpaceType addrSpaceType, bool aslrEnabled, bool aslrDisabled, MemoryRegion memRegion, ulong address, ulong size, KMemoryBlockAllocator blockAllocator) { if ((uint)addrSpaceType > (uint)AddressSpaceType.Addr39Bits) { throw new ArgumentException(nameof(addrSpaceType)); } _contextId = _system.ContextIdManager.GetId(); ulong addrSpaceBase = 0; ulong addrSpaceSize = 1UL << AddrSpaceSizes[(int)addrSpaceType]; KernelResult result = CreateUserAddressSpace( addrSpaceType, aslrEnabled, aslrDisabled, addrSpaceBase, addrSpaceSize, memRegion, address, size, blockAllocator); if (result != KernelResult.Success) { _system.ContextIdManager.PutId(_contextId); } return result; } private class Region { public ulong Start; public ulong End; public ulong Size; public ulong AslrOffset; } private KernelResult CreateUserAddressSpace( AddressSpaceType addrSpaceType, bool aslrEnabled, bool aslrDisabled, ulong addrSpaceStart, ulong addrSpaceEnd, MemoryRegion memRegion, ulong address, ulong size, KMemoryBlockAllocator blockAllocator) { ulong endAddr = address + size; Region aliasRegion = new Region(); Region heapRegion = new Region(); Region stackRegion = new Region(); Region tlsIoRegion = new Region(); ulong codeRegionSize; ulong stackAndTlsIoStart; ulong stackAndTlsIoEnd; ulong baseAddress; switch (addrSpaceType) { case AddressSpaceType.Addr32Bits: aliasRegion.Size = 0x40000000; heapRegion.Size = 0x40000000; stackRegion.Size = 0; tlsIoRegion.Size = 0; CodeRegionStart = 0x200000; codeRegionSize = 0x3fe00000; stackAndTlsIoStart = 0x200000; stackAndTlsIoEnd = 0x40000000; baseAddress = 0x200000; AddrSpaceWidth = 32; break; case AddressSpaceType.Addr36Bits: aliasRegion.Size = 0x180000000; heapRegion.Size = 0x180000000; stackRegion.Size = 0; tlsIoRegion.Size = 0; CodeRegionStart = 0x8000000; codeRegionSize = 0x78000000; stackAndTlsIoStart = 0x8000000; stackAndTlsIoEnd = 0x80000000; baseAddress = 0x8000000; AddrSpaceWidth = 36; break; case AddressSpaceType.Addr32BitsNoMap: aliasRegion.Size = 0; heapRegion.Size = 0x80000000; stackRegion.Size = 0; tlsIoRegion.Size = 0; CodeRegionStart = 0x200000; codeRegionSize = 0x3fe00000; stackAndTlsIoStart = 0x200000; stackAndTlsIoEnd = 0x40000000; baseAddress = 0x200000; AddrSpaceWidth = 32; break; case AddressSpaceType.Addr39Bits: aliasRegion.Size = 0x1000000000; heapRegion.Size = 0x180000000; stackRegion.Size = 0x80000000; tlsIoRegion.Size = 0x1000000000; CodeRegionStart = BitUtils.AlignDown(address, 0x200000); codeRegionSize = BitUtils.AlignUp (endAddr, 0x200000) - CodeRegionStart; stackAndTlsIoStart = 0; stackAndTlsIoEnd = 0; baseAddress = 0x8000000; AddrSpaceWidth = 39; break; default: throw new ArgumentException(nameof(addrSpaceType)); } CodeRegionEnd = CodeRegionStart + codeRegionSize; ulong mapBaseAddress; ulong mapAvailableSize; if (CodeRegionStart - baseAddress >= addrSpaceEnd - CodeRegionEnd) { //Has more space before the start of the code region. mapBaseAddress = baseAddress; mapAvailableSize = CodeRegionStart - baseAddress; } else { //Has more space after the end of the code region. mapBaseAddress = CodeRegionEnd; mapAvailableSize = addrSpaceEnd - CodeRegionEnd; } ulong mapTotalSize = aliasRegion.Size + heapRegion.Size + stackRegion.Size + tlsIoRegion.Size; ulong aslrMaxOffset = mapAvailableSize - mapTotalSize; _aslrEnabled = aslrEnabled; AddrSpaceStart = addrSpaceStart; AddrSpaceEnd = addrSpaceEnd; _blockAllocator = blockAllocator; if (mapAvailableSize < mapTotalSize) { return KernelResult.OutOfMemory; } if (aslrEnabled) { aliasRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; heapRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; stackRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; tlsIoRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; } //Regions are sorted based on ASLR offset. //When ASLR is disabled, the order is Map, Heap, NewMap and TlsIo. aliasRegion.Start = mapBaseAddress + aliasRegion.AslrOffset; aliasRegion.End = aliasRegion.Start + aliasRegion.Size; heapRegion.Start = mapBaseAddress + heapRegion.AslrOffset; heapRegion.End = heapRegion.Start + heapRegion.Size; stackRegion.Start = mapBaseAddress + stackRegion.AslrOffset; stackRegion.End = stackRegion.Start + stackRegion.Size; tlsIoRegion.Start = mapBaseAddress + tlsIoRegion.AslrOffset; tlsIoRegion.End = tlsIoRegion.Start + tlsIoRegion.Size; SortRegion(heapRegion, aliasRegion); if (stackRegion.Size != 0) { SortRegion(stackRegion, aliasRegion); SortRegion(stackRegion, heapRegion); } else { stackRegion.Start = stackAndTlsIoStart; stackRegion.End = stackAndTlsIoEnd; } if (tlsIoRegion.Size != 0) { SortRegion(tlsIoRegion, aliasRegion); SortRegion(tlsIoRegion, heapRegion); SortRegion(tlsIoRegion, stackRegion); } else { tlsIoRegion.Start = stackAndTlsIoStart; tlsIoRegion.End = stackAndTlsIoEnd; } AliasRegionStart = aliasRegion.Start; AliasRegionEnd = aliasRegion.End; HeapRegionStart = heapRegion.Start; HeapRegionEnd = heapRegion.End; StackRegionStart = stackRegion.Start; StackRegionEnd = stackRegion.End; TlsIoRegionStart = tlsIoRegion.Start; TlsIoRegionEnd = tlsIoRegion.End; _currentHeapAddr = HeapRegionStart; _heapCapacity = 0; PhysicalMemoryUsage = 0; _memRegion = memRegion; _aslrDisabled = aslrDisabled; return InitializeBlocks(addrSpaceStart, addrSpaceEnd); } private ulong GetRandomValue(ulong min, ulong max) { return (ulong)GetRandomValue((long)min, (long)max); } private long GetRandomValue(long min, long max) { if (_randomNumberGenerator == null) { _randomNumberGenerator = new MersenneTwister(0); } return _randomNumberGenerator.GenRandomNumber(min, max); } private static void SortRegion(Region lhs, Region rhs) { if (lhs.AslrOffset < rhs.AslrOffset) { rhs.Start += lhs.Size; rhs.End += lhs.Size; } else { lhs.Start += rhs.Size; lhs.End += rhs.Size; } } private KernelResult InitializeBlocks(ulong addrSpaceStart, ulong addrSpaceEnd) { //First insertion will always need only a single block, //because there's nothing else to split. if (!_blockAllocator.CanAllocate(1)) { return KernelResult.OutOfResource; } ulong addrSpacePagesCount = (addrSpaceEnd - addrSpaceStart) / PageSize; InsertBlock(addrSpaceStart, addrSpacePagesCount, MemoryState.Unmapped); return KernelResult.Success; } public KernelResult MapPages( ulong address, KPageList pageList, MemoryState state, MemoryPermission permission) { ulong pagesCount = pageList.GetPagesCount(); ulong size = pagesCount * PageSize; if (!ValidateRegionForState(address, size, state)) { return KernelResult.InvalidMemState; } lock (_blocks) { if (!IsUnmapped(address, pagesCount * PageSize)) { return KernelResult.InvalidMemState; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } KernelResult result = MapPages(address, pageList, permission); if (result == KernelResult.Success) { InsertBlock(address, pagesCount, state, permission); } return result; } } public KernelResult UnmapPages(ulong address, KPageList pageList, MemoryState stateExpected) { ulong pagesCount = pageList.GetPagesCount(); ulong size = pagesCount * PageSize; ulong endAddr = address + size; ulong addrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; if (AddrSpaceStart > address) { return KernelResult.InvalidMemState; } if (addrSpacePagesCount < pagesCount) { return KernelResult.InvalidMemState; } if (endAddr - 1 > AddrSpaceEnd - 1) { return KernelResult.InvalidMemState; } lock (_blocks) { KPageList currentPageList = new KPageList(); AddVaRangeToPageList(currentPageList, address, pagesCount); if (!currentPageList.IsEqual(pageList)) { return KernelResult.InvalidMemRange; } if (CheckRange( address, size, MemoryState.Mask, stateExpected, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState state, out _, out _)) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } KernelResult result = MmuUnmap(address, pagesCount); if (result == KernelResult.Success) { InsertBlock(address, pagesCount, MemoryState.Unmapped); } return result; } else { return KernelResult.InvalidMemState; } } } public KernelResult MapNormalMemory(long address, long size, MemoryPermission permission) { //TODO. return KernelResult.Success; } public KernelResult MapIoMemory(long address, long size, MemoryPermission permission) { //TODO. return KernelResult.Success; } public KernelResult AllocateOrMapPa( ulong neededPagesCount, int alignment, ulong srcPa, bool map, ulong regionStart, ulong regionPagesCount, MemoryState state, MemoryPermission permission, out ulong address) { address = 0; ulong regionSize = regionPagesCount * PageSize; ulong regionEndAddr = regionStart + regionSize; if (!ValidateRegionForState(regionStart, regionSize, state)) { return KernelResult.InvalidMemState; } if (regionPagesCount <= neededPagesCount) { return KernelResult.OutOfMemory; } ulong reservedPagesCount = _isKernel ? 1UL : 4UL; lock (_blocks) { if (_aslrEnabled) { ulong totalNeededSize = (reservedPagesCount + neededPagesCount) * PageSize; ulong remainingPages = regionPagesCount - neededPagesCount; ulong aslrMaxOffset = ((remainingPages + reservedPagesCount) * PageSize) / (ulong)alignment; for (int attempt = 0; attempt < 8; attempt++) { address = BitUtils.AlignDown(regionStart + GetRandomValue(0, aslrMaxOffset) * (ulong)alignment, alignment); ulong endAddr = address + totalNeededSize; KMemoryInfo info = FindBlock(address).GetInfo(); if (info.State != MemoryState.Unmapped) { continue; } ulong currBaseAddr = info.Address + reservedPagesCount * PageSize; ulong currEndAddr = info.Address + info.Size; if (address >= regionStart && address >= currBaseAddr && endAddr - 1 <= regionEndAddr - 1 && endAddr - 1 <= currEndAddr - 1) { break; } } if (address == 0) { ulong aslrPage = GetRandomValue(0, aslrMaxOffset); address = FindFirstFit( regionStart + aslrPage * PageSize, regionPagesCount - aslrPage, neededPagesCount, alignment, 0, reservedPagesCount); } } if (address == 0) { address = FindFirstFit( regionStart, regionPagesCount, neededPagesCount, alignment, 0, reservedPagesCount); } if (address == 0) { return KernelResult.OutOfMemory; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } MemoryOperation operation = map ? MemoryOperation.MapPa : MemoryOperation.Allocate; KernelResult result = DoMmuOperation( address, neededPagesCount, srcPa, map, permission, operation); if (result != KernelResult.Success) { return result; } InsertBlock(address, neededPagesCount, state, permission); } return KernelResult.Success; } public KernelResult MapNewProcessCode( ulong address, ulong pagesCount, MemoryState state, MemoryPermission permission) { ulong size = pagesCount * PageSize; if (!ValidateRegionForState(address, size, state)) { return KernelResult.InvalidMemState; } lock (_blocks) { if (!IsUnmapped(address, size)) { return KernelResult.InvalidMemState; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } KernelResult result = DoMmuOperation( address, pagesCount, 0, false, permission, MemoryOperation.Allocate); if (result == KernelResult.Success) { InsertBlock(address, pagesCount, state, permission); } return result; } } public KernelResult MapProcessCodeMemory(ulong dst, ulong src, ulong size) { ulong pagesCount = size / PageSize; lock (_blocks) { bool success = CheckRange( src, size, MemoryState.Mask, MemoryState.Heap, MemoryPermission.Mask, MemoryPermission.ReadAndWrite, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState state, out MemoryPermission permission, out _); success &= IsUnmapped(dst, size); if (success) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) { return KernelResult.OutOfResource; } KPageList pageList = new KPageList(); AddVaRangeToPageList(pageList, src, pagesCount); KernelResult result = MmuChangePermission(src, pagesCount, MemoryPermission.None); if (result != KernelResult.Success) { return result; } result = MapPages(dst, pageList, MemoryPermission.None); if (result != KernelResult.Success) { MmuChangePermission(src, pagesCount, permission); return result; } InsertBlock(src, pagesCount, state, MemoryPermission.None, MemoryAttribute.Borrowed); InsertBlock(dst, pagesCount, MemoryState.ModCodeStatic); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult UnmapProcessCodeMemory(ulong dst, ulong src, ulong size) { ulong pagesCount = size / PageSize; lock (_blocks) { bool success = CheckRange( src, size, MemoryState.Mask, MemoryState.Heap, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.Borrowed, MemoryAttribute.IpcAndDeviceMapped, out _, out _, out _); success &= CheckRange( dst, PageSize, MemoryState.UnmapProcessCodeMemoryAllowed, MemoryState.UnmapProcessCodeMemoryAllowed, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState state, out _, out _); success &= CheckRange( dst, size, MemoryState.Mask, state, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None); if (success) { KernelResult result = MmuUnmap(dst, pagesCount); if (result != KernelResult.Success) { return result; } //TODO: Missing some checks here. if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) { return KernelResult.OutOfResource; } InsertBlock(dst, pagesCount, MemoryState.Unmapped); InsertBlock(src, pagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult SetHeapSize(ulong size, out ulong address) { address = 0; if (size > HeapRegionEnd - HeapRegionStart) { return KernelResult.OutOfMemory; } KProcess currentProcess = _system.Scheduler.GetCurrentProcess(); ulong currentHeapSize = GetHeapSize(); if (currentHeapSize <= size) { //Expand. ulong diffSize = size - currentHeapSize; lock (_blocks) { if (currentProcess.ResourceLimit != null && diffSize != 0 && !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, diffSize)) { return KernelResult.ResLimitExceeded; } ulong pagesCount = diffSize / PageSize; KMemoryRegionManager region = GetMemoryRegionManager(); KernelResult result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); void CleanUpForError() { if (pageList != null) { region.FreePages(pageList); } if (currentProcess.ResourceLimit != null && diffSize != 0) { currentProcess.ResourceLimit.Release(LimitableResource.Memory, diffSize); } } if (result != KernelResult.Success) { CleanUpForError(); return result; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { CleanUpForError(); return KernelResult.OutOfResource; } if (!IsUnmapped(_currentHeapAddr, diffSize)) { CleanUpForError(); return KernelResult.InvalidMemState; } result = DoMmuOperation( _currentHeapAddr, pagesCount, pageList, MemoryPermission.ReadAndWrite, MemoryOperation.MapVa); if (result != KernelResult.Success) { CleanUpForError(); return result; } InsertBlock(_currentHeapAddr, pagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); } } else { //Shrink. ulong freeAddr = HeapRegionStart + size; ulong diffSize = currentHeapSize - size; lock (_blocks) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } if (!CheckRange( freeAddr, diffSize, MemoryState.Mask, MemoryState.Heap, MemoryPermission.Mask, MemoryPermission.ReadAndWrite, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out _, out _, out _)) { return KernelResult.InvalidMemState; } ulong pagesCount = diffSize / PageSize; KernelResult result = MmuUnmap(freeAddr, pagesCount); if (result != KernelResult.Success) { return result; } currentProcess.ResourceLimit?.Release(LimitableResource.Memory, BitUtils.AlignDown(diffSize, PageSize)); InsertBlock(freeAddr, pagesCount, MemoryState.Unmapped); } } _currentHeapAddr = HeapRegionStart + size; address = HeapRegionStart; return KernelResult.Success; } public ulong GetTotalHeapSize() { lock (_blocks) { return GetHeapSize() + PhysicalMemoryUsage; } } private ulong GetHeapSize() { return _currentHeapAddr - HeapRegionStart; } public KernelResult SetHeapCapacity(ulong capacity) { lock (_blocks) { _heapCapacity = capacity; } return KernelResult.Success; } public KernelResult SetMemoryAttribute( ulong address, ulong size, MemoryAttribute attributeMask, MemoryAttribute attributeValue) { lock (_blocks) { if (CheckRange( address, size, MemoryState.AttributeChangeAllowed, MemoryState.AttributeChangeAllowed, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.BorrowedAndIpcMapped, MemoryAttribute.None, MemoryAttribute.DeviceMappedAndUncached, out MemoryState state, out MemoryPermission permission, out MemoryAttribute attribute)) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; attribute &= ~attributeMask; attribute |= attributeMask & attributeValue; InsertBlock(address, pagesCount, state, permission, attribute); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KMemoryInfo QueryMemory(ulong address) { if (address >= AddrSpaceStart && address < AddrSpaceEnd) { lock (_blocks) { return FindBlock(address).GetInfo(); } } else { return new KMemoryInfo( AddrSpaceEnd, ~AddrSpaceEnd + 1, MemoryState.Reserved, MemoryPermission.None, MemoryAttribute.None, 0, 0); } } public KernelResult Map(ulong dst, ulong src, ulong size) { bool success; lock (_blocks) { success = CheckRange( src, size, MemoryState.MapAllowed, MemoryState.MapAllowed, MemoryPermission.Mask, MemoryPermission.ReadAndWrite, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState srcState, out _, out _); success &= IsUnmapped(dst, size); if (success) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; KPageList pageList = new KPageList(); AddVaRangeToPageList(pageList, src, pagesCount); KernelResult result = MmuChangePermission(src, pagesCount, MemoryPermission.None); if (result != KernelResult.Success) { return result; } result = MapPages(dst, pageList, MemoryPermission.ReadAndWrite); if (result != KernelResult.Success) { if (MmuChangePermission(src, pagesCount, MemoryPermission.ReadAndWrite) != KernelResult.Success) { throw new InvalidOperationException("Unexpected failure reverting memory permission."); } return result; } InsertBlock(src, pagesCount, srcState, MemoryPermission.None, MemoryAttribute.Borrowed); InsertBlock(dst, pagesCount, MemoryState.Stack, MemoryPermission.ReadAndWrite); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult UnmapForKernel(ulong address, ulong pagesCount, MemoryState stateExpected) { ulong size = pagesCount * PageSize; lock (_blocks) { if (CheckRange( address, size, MemoryState.Mask, stateExpected, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out _, out _, out _)) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } KernelResult result = MmuUnmap(address, pagesCount); if (result == KernelResult.Success) { InsertBlock(address, pagesCount, MemoryState.Unmapped); } return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult Unmap(ulong dst, ulong src, ulong size) { bool success; lock (_blocks) { success = CheckRange( src, size, MemoryState.MapAllowed, MemoryState.MapAllowed, MemoryPermission.Mask, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.Borrowed, MemoryAttribute.IpcAndDeviceMapped, out MemoryState srcState, out _, out _); success &= CheckRange( dst, size, MemoryState.Mask, MemoryState.Stack, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out _, out MemoryPermission dstPermission, out _); if (success) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; KPageList srcPageList = new KPageList(); KPageList dstPageList = new KPageList(); AddVaRangeToPageList(srcPageList, src, pagesCount); AddVaRangeToPageList(dstPageList, dst, pagesCount); if (!dstPageList.IsEqual(srcPageList)) { return KernelResult.InvalidMemRange; } KernelResult result = MmuUnmap(dst, pagesCount); if (result != KernelResult.Success) { return result; } result = MmuChangePermission(src, pagesCount, MemoryPermission.ReadAndWrite); if (result != KernelResult.Success) { MapPages(dst, dstPageList, dstPermission); return result; } InsertBlock(src, pagesCount, srcState, MemoryPermission.ReadAndWrite); InsertBlock(dst, pagesCount, MemoryState.Unmapped); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult ReserveTransferMemory(ulong address, ulong size, MemoryPermission permission) { lock (_blocks) { if (CheckRange( address, size, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryPermission.Mask, MemoryPermission.ReadAndWrite, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState state, out _, out MemoryAttribute attribute)) { //TODO: Missing checks. if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; attribute |= MemoryAttribute.Borrowed; InsertBlock(address, pagesCount, state, permission, attribute); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult ResetTransferMemory(ulong address, ulong size) { lock (_blocks) { if (CheckRange( address, size, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.Borrowed, MemoryAttribute.IpcAndDeviceMapped, out MemoryState state, out _, out _)) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; InsertBlock(address, pagesCount, state, MemoryPermission.ReadAndWrite); return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult SetProcessMemoryPermission(ulong address, ulong size, MemoryPermission permission) { lock (_blocks) { if (CheckRange( address, size, MemoryState.ProcessPermissionChangeAllowed, MemoryState.ProcessPermissionChangeAllowed, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out MemoryState oldState, out MemoryPermission oldPermission, out _)) { MemoryState newState = oldState; //If writing into the code region is allowed, then we need //to change it to mutable. if ((permission & MemoryPermission.Write) != 0) { if (oldState == MemoryState.CodeStatic) { newState = MemoryState.CodeMutable; } else if (oldState == MemoryState.ModCodeStatic) { newState = MemoryState.ModCodeMutable; } else { throw new InvalidOperationException($"Memory state \"{oldState}\" not valid for this operation."); } } if (newState != oldState || permission != oldPermission) { if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } ulong pagesCount = size / PageSize; MemoryOperation operation = (permission & MemoryPermission.Execute) != 0 ? MemoryOperation.ChangePermsAndAttributes : MemoryOperation.ChangePermRw; KernelResult result = DoMmuOperation(address, pagesCount, 0, false, permission, operation); if (result != KernelResult.Success) { return result; } InsertBlock(address, pagesCount, newState, permission); } return KernelResult.Success; } else { return KernelResult.InvalidMemState; } } } public KernelResult MapPhysicalMemory(ulong address, ulong size) { ulong endAddr = address + size; lock (_blocks) { ulong mappedSize = 0; KMemoryInfo info; LinkedListNode node = FindBlockNode(address); do { info = node.Value.GetInfo(); if (info.State != MemoryState.Unmapped) { mappedSize += GetSizeInRange(info, address, endAddr); } node = node.Next; } while (info.Address + info.Size < endAddr && node != null); if (mappedSize == size) { return KernelResult.Success; } ulong remainingSize = size - mappedSize; ulong remainingPages = remainingSize / PageSize; KProcess currentProcess = _system.Scheduler.GetCurrentProcess(); if (currentProcess.ResourceLimit != null && !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, remainingSize)) { return KernelResult.ResLimitExceeded; } KMemoryRegionManager region = GetMemoryRegionManager(); KernelResult result = region.AllocatePages(remainingPages, _aslrDisabled, out KPageList pageList); void CleanUpForError() { if (pageList != null) { region.FreePages(pageList); } currentProcess.ResourceLimit?.Release(LimitableResource.Memory, remainingSize); } if (result != KernelResult.Success) { CleanUpForError(); return result; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { CleanUpForError(); return KernelResult.OutOfResource; } MapPhysicalMemory(pageList, address, endAddr); PhysicalMemoryUsage += remainingSize; ulong pagesCount = size / PageSize; InsertBlock( address, pagesCount, MemoryState.Unmapped, MemoryPermission.None, MemoryAttribute.None, MemoryState.Heap, MemoryPermission.ReadAndWrite, MemoryAttribute.None); } return KernelResult.Success; } public KernelResult UnmapPhysicalMemory(ulong address, ulong size) { ulong endAddr = address + size; lock (_blocks) { //Scan, ensure that the region can be unmapped (all blocks are heap or //already unmapped), fill pages list for freeing memory. ulong heapMappedSize = 0; KPageList pageList = new KPageList(); KMemoryInfo info; LinkedListNode baseNode = FindBlockNode(address); LinkedListNode node = baseNode; do { info = node.Value.GetInfo(); if (info.State == MemoryState.Heap) { if (info.Attribute != MemoryAttribute.None) { return KernelResult.InvalidMemState; } ulong blockSize = GetSizeInRange(info, address, endAddr); ulong blockAddress = GetAddrInRange(info, address); AddVaRangeToPageList(pageList, blockAddress, blockSize / PageSize); heapMappedSize += blockSize; } else if (info.State != MemoryState.Unmapped) { return KernelResult.InvalidMemState; } node = node.Next; } while (info.Address + info.Size < endAddr && node != null); if (heapMappedSize == 0) { return KernelResult.Success; } if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) { return KernelResult.OutOfResource; } //Try to unmap all the heap mapped memory inside range. KernelResult result = KernelResult.Success; node = baseNode; do { info = node.Value.GetInfo(); if (info.State == MemoryState.Heap) { ulong blockSize = GetSizeInRange(info, address, endAddr); ulong blockAddress = GetAddrInRange(info, address); ulong blockPagesCount = blockSize / PageSize; result = MmuUnmap(blockAddress, blockPagesCount); if (result != KernelResult.Success) { //If we failed to unmap, we need to remap everything back again. MapPhysicalMemory(pageList, address, blockAddress + blockSize); break; } } node = node.Next; } while (info.Address + info.Size < endAddr && node != null); if (result == KernelResult.Success) { GetMemoryRegionManager().FreePages(pageList); PhysicalMemoryUsage -= heapMappedSize; KProcess currentProcess = _system.Scheduler.GetCurrentProcess(); currentProcess.ResourceLimit?.Release(LimitableResource.Memory, heapMappedSize); ulong pagesCount = size / PageSize; InsertBlock(address, pagesCount, MemoryState.Unmapped); } return result; } } private void MapPhysicalMemory(KPageList pageList, ulong address, ulong endAddr) { KMemoryInfo info; LinkedListNode node = FindBlockNode(address); LinkedListNode pageListNode = pageList.Nodes.First; KPageNode pageNode = pageListNode.Value; ulong srcPa = pageNode.Address; ulong srcPaPages = pageNode.PagesCount; do { info = node.Value.GetInfo(); if (info.State == MemoryState.Unmapped) { ulong blockSize = GetSizeInRange(info, address, endAddr); ulong dstVaPages = blockSize / PageSize; ulong dstVa = GetAddrInRange(info, address); while (dstVaPages > 0) { if (srcPaPages == 0) { pageListNode = pageListNode.Next; pageNode = pageListNode.Value; srcPa = pageNode.Address; srcPaPages = pageNode.PagesCount; } ulong pagesCount = srcPaPages; if (pagesCount > dstVaPages) { pagesCount = dstVaPages; } DoMmuOperation( dstVa, pagesCount, srcPa, true, MemoryPermission.ReadAndWrite, MemoryOperation.MapPa); dstVa += pagesCount * PageSize; srcPa += pagesCount * PageSize; srcPaPages -= pagesCount; dstVaPages -= pagesCount; } } node = node.Next; } while (info.Address + info.Size < endAddr && node != null); } private static ulong GetSizeInRange(KMemoryInfo info, ulong start, ulong end) { ulong endAddr = info.Size + info.Address; ulong size = info.Size; if (info.Address < start) { size -= start - info.Address; } if (endAddr > end) { size -= endAddr - end; } return size; } private static ulong GetAddrInRange(KMemoryInfo info, ulong start) { if (info.Address < start) { return start; } return info.Address; } private void AddVaRangeToPageList(KPageList pageList, ulong start, ulong pagesCount) { ulong address = start; while (address < start + pagesCount * PageSize) { KernelResult result = ConvertVaToPa(address, out ulong pa); if (result != KernelResult.Success) { throw new InvalidOperationException("Unexpected failure translating virtual address."); } pageList.AddRange(pa, 1); address += PageSize; } } private bool IsUnmapped(ulong address, ulong size) { return CheckRange( address, size, MemoryState.Mask, MemoryState.Unmapped, MemoryPermission.Mask, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out _, out _, out _); } private bool CheckRange( ulong address, ulong size, MemoryState stateMask, MemoryState stateExpected, MemoryPermission permissionMask, MemoryPermission permissionExpected, MemoryAttribute attributeMask, MemoryAttribute attributeExpected, MemoryAttribute attributeIgnoreMask, out MemoryState outState, out MemoryPermission outPermission, out MemoryAttribute outAttribute) { ulong endAddr = address + size - 1; LinkedListNode node = FindBlockNode(address); KMemoryInfo info = node.Value.GetInfo(); MemoryState firstState = info.State; MemoryPermission firstPermission = info.Permission; MemoryAttribute firstAttribute = info.Attribute; do { info = node.Value.GetInfo(); //Check if the block state matches what we expect. if ( firstState != info.State || firstPermission != info.Permission || (info.Attribute & attributeMask) != attributeExpected || (firstAttribute | attributeIgnoreMask) != (info.Attribute | attributeIgnoreMask) || (firstState & stateMask) != stateExpected || (firstPermission & permissionMask) != permissionExpected) { break; } //Check if this is the last block on the range, if so return success. if (endAddr <= info.Address + info.Size - 1) { outState = firstState; outPermission = firstPermission; outAttribute = firstAttribute & ~attributeIgnoreMask; return true; } node = node.Next; } while (node != null); outState = MemoryState.Unmapped; outPermission = MemoryPermission.None; outAttribute = MemoryAttribute.None; return false; } private bool CheckRange( ulong address, ulong size, MemoryState stateMask, MemoryState stateExpected, MemoryPermission permissionMask, MemoryPermission permissionExpected, MemoryAttribute attributeMask, MemoryAttribute attributeExpected) { ulong endAddr = address + size - 1; LinkedListNode node = FindBlockNode(address); do { KMemoryInfo info = node.Value.GetInfo(); //Check if the block state matches what we expect. if ((info.State & stateMask) != stateExpected || (info.Permission & permissionMask) != permissionExpected || (info.Attribute & attributeMask) != attributeExpected) { break; } //Check if this is the last block on the range, if so return success. if (endAddr <= info.Address + info.Size - 1) { return true; } node = node.Next; } while (node != null); return false; } private void InsertBlock( ulong baseAddress, ulong pagesCount, MemoryState oldState, MemoryPermission oldPermission, MemoryAttribute oldAttribute, MemoryState newState, MemoryPermission newPermission, MemoryAttribute newAttribute) { //Insert new block on the list only on areas where the state //of the block matches the state specified on the Old* state //arguments, otherwise leave it as is. int oldCount = _blocks.Count; oldAttribute |= MemoryAttribute.IpcAndDeviceMapped; ulong endAddr = pagesCount * PageSize + baseAddress; LinkedListNode node = _blocks.First; while (node != null) { LinkedListNode newNode = node; LinkedListNode nextNode = node.Next; KMemoryBlock currBlock = node.Value; ulong currBaseAddr = currBlock.BaseAddress; ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; if (baseAddress < currEndAddr && currBaseAddr < endAddr) { MemoryAttribute currBlockAttr = currBlock.Attribute | MemoryAttribute.IpcAndDeviceMapped; if (currBlock.State != oldState || currBlock.Permission != oldPermission || currBlockAttr != oldAttribute) { node = nextNode; continue; } if (currBaseAddr >= baseAddress && currEndAddr <= endAddr) { currBlock.State = newState; currBlock.Permission = newPermission; currBlock.Attribute &= ~MemoryAttribute.IpcAndDeviceMapped; currBlock.Attribute |= newAttribute; } else if (currBaseAddr >= baseAddress) { currBlock.BaseAddress = endAddr; currBlock.PagesCount = (currEndAddr - endAddr) / PageSize; ulong newPagesCount = (endAddr - currBaseAddr) / PageSize; newNode = _blocks.AddBefore(node, new KMemoryBlock( currBaseAddr, newPagesCount, newState, newPermission, newAttribute)); } else if (currEndAddr <= endAddr) { currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; ulong newPagesCount = (currEndAddr - baseAddress) / PageSize; newNode = _blocks.AddAfter(node, new KMemoryBlock( baseAddress, newPagesCount, newState, newPermission, newAttribute)); } else { currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; ulong nextPagesCount = (currEndAddr - endAddr) / PageSize; newNode = _blocks.AddAfter(node, new KMemoryBlock( baseAddress, pagesCount, newState, newPermission, newAttribute)); _blocks.AddAfter(newNode, new KMemoryBlock( endAddr, nextPagesCount, currBlock.State, currBlock.Permission, currBlock.Attribute)); nextNode = null; } MergeEqualStateNeighbours(newNode); } node = nextNode; } _blockAllocator.Count += _blocks.Count - oldCount; } private void InsertBlock( ulong baseAddress, ulong pagesCount, MemoryState state, MemoryPermission permission = MemoryPermission.None, MemoryAttribute attribute = MemoryAttribute.None) { //Inserts new block at the list, replacing and spliting //existing blocks as needed. KMemoryBlock block = new KMemoryBlock(baseAddress, pagesCount, state, permission, attribute); int oldCount = _blocks.Count; ulong endAddr = pagesCount * PageSize + baseAddress; LinkedListNode newNode = null; LinkedListNode node = _blocks.First; while (node != null) { KMemoryBlock currBlock = node.Value; LinkedListNode nextNode = node.Next; ulong currBaseAddr = currBlock.BaseAddress; ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; if (baseAddress < currEndAddr && currBaseAddr < endAddr) { if (baseAddress >= currBaseAddr && endAddr <= currEndAddr) { block.Attribute |= currBlock.Attribute & MemoryAttribute.IpcAndDeviceMapped; } if (baseAddress > currBaseAddr && endAddr < currEndAddr) { currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; ulong nextPagesCount = (currEndAddr - endAddr) / PageSize; newNode = _blocks.AddAfter(node, block); _blocks.AddAfter(newNode, new KMemoryBlock( endAddr, nextPagesCount, currBlock.State, currBlock.Permission, currBlock.Attribute)); break; } else if (baseAddress <= currBaseAddr && endAddr < currEndAddr) { currBlock.BaseAddress = endAddr; currBlock.PagesCount = (currEndAddr - endAddr) / PageSize; if (newNode == null) { newNode = _blocks.AddBefore(node, block); } } else if (baseAddress > currBaseAddr && endAddr >= currEndAddr) { currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; if (newNode == null) { newNode = _blocks.AddAfter(node, block); } } else { if (newNode == null) { newNode = _blocks.AddBefore(node, block); } _blocks.Remove(node); } } node = nextNode; } if (newNode == null) { newNode = _blocks.AddFirst(block); } MergeEqualStateNeighbours(newNode); _blockAllocator.Count += _blocks.Count - oldCount; } private void MergeEqualStateNeighbours(LinkedListNode node) { KMemoryBlock block = node.Value; ulong endAddr = block.PagesCount * PageSize + block.BaseAddress; if (node.Previous != null) { KMemoryBlock previous = node.Previous.Value; if (BlockStateEquals(block, previous)) { _blocks.Remove(node.Previous); block.BaseAddress = previous.BaseAddress; } } if (node.Next != null) { KMemoryBlock next = node.Next.Value; if (BlockStateEquals(block, next)) { _blocks.Remove(node.Next); endAddr = next.BaseAddress + next.PagesCount * PageSize; } } block.PagesCount = (endAddr - block.BaseAddress) / PageSize; } private static bool BlockStateEquals(KMemoryBlock lhs, KMemoryBlock rhs) { return lhs.State == rhs.State && lhs.Permission == rhs.Permission && lhs.Attribute == rhs.Attribute && lhs.DeviceRefCount == rhs.DeviceRefCount && lhs.IpcRefCount == rhs.IpcRefCount; } private ulong FindFirstFit( ulong regionStart, ulong regionPagesCount, ulong neededPagesCount, int alignment, ulong reservedStart, ulong reservedPagesCount) { ulong reservedSize = reservedPagesCount * PageSize; ulong totalNeededSize = reservedSize + neededPagesCount * PageSize; ulong regionEndAddr = regionStart + regionPagesCount * PageSize; LinkedListNode node = FindBlockNode(regionStart); KMemoryInfo info = node.Value.GetInfo(); while (regionEndAddr >= info.Address) { if (info.State == MemoryState.Unmapped) { ulong currBaseAddr = info.Address + reservedSize; ulong currEndAddr = info.Address + info.Size - 1; ulong address = BitUtils.AlignDown(currBaseAddr, alignment) + reservedStart; if (currBaseAddr > address) { address += (ulong)alignment; } ulong allocationEndAddr = address + totalNeededSize - 1; if (allocationEndAddr <= regionEndAddr && allocationEndAddr <= currEndAddr && address < allocationEndAddr) { return address; } } node = node.Next; if (node == null) { break; } info = node.Value.GetInfo(); } return 0; } private KMemoryBlock FindBlock(ulong address) { return FindBlockNode(address)?.Value; } private LinkedListNode FindBlockNode(ulong address) { lock (_blocks) { LinkedListNode node = _blocks.First; while (node != null) { KMemoryBlock block = node.Value; ulong currEndAddr = block.PagesCount * PageSize + block.BaseAddress; if (block.BaseAddress <= address && currEndAddr - 1 >= address) { return node; } node = node.Next; } } return null; } private bool ValidateRegionForState(ulong address, ulong size, MemoryState state) { ulong endAddr = address + size; ulong regionBaseAddr = GetBaseAddrForState(state); ulong regionEndAddr = regionBaseAddr + GetSizeForState(state); bool InsideRegion() { return regionBaseAddr <= address && endAddr > address && endAddr - 1 <= regionEndAddr - 1; } bool OutsideHeapRegion() { return endAddr <= HeapRegionStart || address >= HeapRegionEnd; } bool OutsideMapRegion() { return endAddr <= AliasRegionStart || address >= AliasRegionEnd; } switch (state) { case MemoryState.Io: case MemoryState.Normal: case MemoryState.CodeStatic: case MemoryState.CodeMutable: case MemoryState.SharedMemory: case MemoryState.ModCodeStatic: case MemoryState.ModCodeMutable: case MemoryState.Stack: case MemoryState.ThreadLocal: case MemoryState.TransferMemoryIsolated: case MemoryState.TransferMemory: case MemoryState.ProcessMemory: case MemoryState.CodeReadOnly: case MemoryState.CodeWritable: return InsideRegion() && OutsideHeapRegion() && OutsideMapRegion(); case MemoryState.Heap: return InsideRegion() && OutsideMapRegion(); case MemoryState.IpcBuffer0: case MemoryState.IpcBuffer1: case MemoryState.IpcBuffer3: return InsideRegion() && OutsideHeapRegion(); case MemoryState.KernelStack: return InsideRegion(); } throw new ArgumentException($"Invalid state value \"{state}\"."); } private ulong GetBaseAddrForState(MemoryState state) { switch (state) { case MemoryState.Io: case MemoryState.Normal: case MemoryState.ThreadLocal: return TlsIoRegionStart; case MemoryState.CodeStatic: case MemoryState.CodeMutable: case MemoryState.SharedMemory: case MemoryState.ModCodeStatic: case MemoryState.ModCodeMutable: case MemoryState.TransferMemoryIsolated: case MemoryState.TransferMemory: case MemoryState.ProcessMemory: case MemoryState.CodeReadOnly: case MemoryState.CodeWritable: return GetAddrSpaceBaseAddr(); case MemoryState.Heap: return HeapRegionStart; case MemoryState.IpcBuffer0: case MemoryState.IpcBuffer1: case MemoryState.IpcBuffer3: return AliasRegionStart; case MemoryState.Stack: return StackRegionStart; case MemoryState.KernelStack: return AddrSpaceStart; } throw new ArgumentException($"Invalid state value \"{state}\"."); } private ulong GetSizeForState(MemoryState state) { switch (state) { case MemoryState.Io: case MemoryState.Normal: case MemoryState.ThreadLocal: return TlsIoRegionEnd - TlsIoRegionStart; case MemoryState.CodeStatic: case MemoryState.CodeMutable: case MemoryState.SharedMemory: case MemoryState.ModCodeStatic: case MemoryState.ModCodeMutable: case MemoryState.TransferMemoryIsolated: case MemoryState.TransferMemory: case MemoryState.ProcessMemory: case MemoryState.CodeReadOnly: case MemoryState.CodeWritable: return GetAddrSpaceSize(); case MemoryState.Heap: return HeapRegionEnd - HeapRegionStart; case MemoryState.IpcBuffer0: case MemoryState.IpcBuffer1: case MemoryState.IpcBuffer3: return AliasRegionEnd - AliasRegionStart; case MemoryState.Stack: return StackRegionEnd - StackRegionStart; case MemoryState.KernelStack: return AddrSpaceEnd - AddrSpaceStart; } throw new ArgumentException($"Invalid state value \"{state}\"."); } public ulong GetAddrSpaceBaseAddr() { if (AddrSpaceWidth == 36 || AddrSpaceWidth == 39) { return 0x8000000; } else if (AddrSpaceWidth == 32) { return 0x200000; } else { throw new InvalidOperationException("Invalid address space width!"); } } public ulong GetAddrSpaceSize() { if (AddrSpaceWidth == 36) { return 0xff8000000; } else if (AddrSpaceWidth == 39) { return 0x7ff8000000; } else if (AddrSpaceWidth == 32) { return 0xffe00000; } else { throw new InvalidOperationException("Invalid address space width!"); } } private KernelResult MapPages(ulong address, KPageList pageList, MemoryPermission permission) { ulong currAddr = address; KernelResult result = KernelResult.Success; foreach (KPageNode pageNode in pageList) { result = DoMmuOperation( currAddr, pageNode.PagesCount, pageNode.Address, true, permission, MemoryOperation.MapPa); if (result != KernelResult.Success) { KMemoryInfo info = FindBlock(currAddr).GetInfo(); ulong pagesCount = (address - currAddr) / PageSize; result = MmuUnmap(address, pagesCount); break; } currAddr += pageNode.PagesCount * PageSize; } return result; } private KernelResult MmuUnmap(ulong address, ulong pagesCount) { return DoMmuOperation( address, pagesCount, 0, false, MemoryPermission.None, MemoryOperation.Unmap); } private KernelResult MmuChangePermission(ulong address, ulong pagesCount, MemoryPermission permission) { return DoMmuOperation( address, pagesCount, 0, false, permission, MemoryOperation.ChangePermRw); } private KernelResult DoMmuOperation( ulong dstVa, ulong pagesCount, ulong srcPa, bool map, MemoryPermission permission, MemoryOperation operation) { if (map != (operation == MemoryOperation.MapPa)) { throw new ArgumentException(nameof(map) + " value is invalid for this operation."); } KernelResult result; switch (operation) { case MemoryOperation.MapPa: { ulong size = pagesCount * PageSize; _cpuMemory.Map((long)dstVa, (long)(srcPa - DramMemoryMap.DramBase), (long)size); result = KernelResult.Success; break; } case MemoryOperation.Allocate: { KMemoryRegionManager region = GetMemoryRegionManager(); result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); if (result == KernelResult.Success) { result = MmuMapPages(dstVa, pageList); } break; } case MemoryOperation.Unmap: { ulong size = pagesCount * PageSize; _cpuMemory.Unmap((long)dstVa, (long)size); result = KernelResult.Success; break; } case MemoryOperation.ChangePermRw: result = KernelResult.Success; break; case MemoryOperation.ChangePermsAndAttributes: result = KernelResult.Success; break; default: throw new ArgumentException($"Invalid operation \"{operation}\"."); } return result; } private KernelResult DoMmuOperation( ulong address, ulong pagesCount, KPageList pageList, MemoryPermission permission, MemoryOperation operation) { if (operation != MemoryOperation.MapVa) { throw new ArgumentException($"Invalid memory operation \"{operation}\" specified."); } return MmuMapPages(address, pageList); } private KMemoryRegionManager GetMemoryRegionManager() { return _system.MemoryRegions[(int)_memRegion]; } private KernelResult MmuMapPages(ulong address, KPageList pageList) { foreach (KPageNode pageNode in pageList) { ulong size = pageNode.PagesCount * PageSize; _cpuMemory.Map((long)address, (long)(pageNode.Address - DramMemoryMap.DramBase), (long)size); address += size; } return KernelResult.Success; } public KernelResult ConvertVaToPa(ulong va, out ulong pa) { pa = DramMemoryMap.DramBase + (ulong)_cpuMemory.GetPhysicalAddress((long)va); return KernelResult.Success; } public long GetMmUsedPages() { lock (_blocks) { return BitUtils.DivRoundUp(GetMmUsedSize(), PageSize); } } private long GetMmUsedSize() { return _blocks.Count * KMemoryBlockSize; } public bool IsInvalidRegion(ulong address, ulong size) { return address + size - 1 > GetAddrSpaceBaseAddr() + GetAddrSpaceSize() - 1; } public bool InsideAddrSpace(ulong address, ulong size) { return AddrSpaceStart <= address && address + size - 1 <= AddrSpaceEnd - 1; } public bool InsideAliasRegion(ulong address, ulong size) { return address + size > AliasRegionStart && AliasRegionEnd > address; } public bool InsideHeapRegion(ulong address, ulong size) { return address + size > HeapRegionStart && HeapRegionEnd > address; } public bool InsideStackRegion(ulong address, ulong size) { return address + size > StackRegionStart && StackRegionEnd > address; } public bool OutsideAliasRegion(ulong address, ulong size) { return AliasRegionStart > address || address + size - 1 > AliasRegionEnd - 1; } public bool OutsideAddrSpace(ulong address, ulong size) { return AddrSpaceStart > address || address + size - 1 > AddrSpaceEnd - 1; } public bool OutsideStackRegion(ulong address, ulong size) { return StackRegionStart > address || address + size - 1 > StackRegionEnd - 1; } } }