大型建筑网站设计公司详情外链工厂 外链

在程序的应用开发时候，面对内存密集型操作时，需要大量内存，可能需要远超物理内存空间的内存，该怎么做呢？
假设现在的机器是64位的windows，其用户的虚地址空间是0x000’000000000到0x7FFF’FFFFFFFF，约128 TB。

可以考虑提前申请空间到数据段

#include <iostream>
constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;
uint8_t gBuff[10 * GB_];
int main()
{return 0;
}

这样可以得到一个编译错误

错误	C2148	数组的总大小不得超过 0x7fffffff 字节

可以考虑使用malloc动态申请内存
malloc会申请到一块连续的内存，如果失败，会返回空指针

#include "stdafx.h"
#include <iostream>constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;int main()
{void* ptr = malloc(48 * GB_);if (!ptr) {std::cerr << strerror(errno) << std::endl;exit(EXIT_FAILURE);}return 0;
}

这样的话就只能退一步，分批申请小块内存，一直到申请内存的总量是达到预期。

#include "stdafx.h"
#include <iostream>constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;void* MyMalloc(const size_t expect, size_t& actual)
{actual = expect;void *ptr = nullptr;while (actual && !(ptr = malloc(actual))){actual = actual>>1;}return ptr;
}int main()
{size_t expect = 48 * GB_;constexpr int len = 0xffff;void* buff[len];int i = 0;for (i = 0; i < len && expect; ++i){size_t actual;buff[i] = MyMalloc(expect, actual);if (buff[i] && actual){expect -= actual;std::cout << "malloced (" << actual << ") = " << buff[i] << std::endl;}else {break;}}if (expect) {std::cerr << "malloc big memory fail!" << std::endl;}for (i = 0; i < len && buff[i]; ++i){free(buff[i]);buff[i] = nullptr;}return 0;
}

可能会得到一个崩溃

windows上的malloc可能是使用HeapAlloc或VirtualAlloc实现的，内存不够用的时候发生崩溃。

需要借助VirualAlloc
具体查看 Windows虚拟内存函数

#include <iostream>
#include <windows.h>constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;void* MyAlloc(const size_t expect, size_t& actual)
{actual = expect;void *ptr = nullptr;while (actual && !(ptr = VirtualAlloc(nullptr, actual, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE))){actual = actual >> 1;}return ptr;
}bool MyFree(void* ptr)
{return (TRUE == VirtualFree(ptr, 0, MEM_RELEASE));
}int main()
{size_t expect = 48 * GB_;constexpr int len = 0xffff;void* buff[len];int i = 0;for (i = 0; i < len && expect; ++i){size_t actual;buff[i] = MyAlloc(expect, actual);if (buff[i] && actual){expect -= actual;std::cout << "myalloced (" << actual << ") = " << buff[i] << std::endl;}else {break;}}if (expect) {std::cerr << "alloc big memory fail!" << std::endl;}else {std::cout << "alloced success" << std::endl;}for (i = 0; i < len && buff[i]; ++i){MyFree(buff[i]);buff[i] = nullptr;}return 0;
}

虚拟内存可能在不使用的时候暂存到硬盘交换区中去，可以使用MEM_RESET与MEM_RESET_UNDO来操作。

#include "stdafx.h"
#include <iostream>
#include <cassert>
#include <windows.h>constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;constexpr size_t len = 0xffff;
uint8_t* buff[len];size_t MyCommit(void* const basePtr, const size_t expect)
{size_t actual = expect;while (actual && !VirtualAlloc(basePtr, actual, MEM_COMMIT, PAGE_READWRITE)){actual >>= 1;}if (actual) {MEMORY_BASIC_INFORMATION memInfo;VirtualQuery(basePtr, &memInfo, sizeof(memInfo));if (memInfo.BaseAddress == basePtr && memInfo.State == MEM_COMMIT) {actual = memInfo.RegionSize;}else {actual = 0;}}return actual;
}int main()
{MEMORY_BASIC_INFORMATION memInfo;constexpr size_t expect = 64 * GB_;// 无法直接申请到超大块内存，需要先预订内存void* ptr = VirtualAlloc(nullptr, expect, MEM_RESERVE, PAGE_NOACCESS);VirtualQuery(ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);// 然后申请内存const size_t actual = MyCommit(ptr, expect);assert(actual);VirtualQuery(ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);// 在申请到内存内写入数据for (int i = 0; i < 10; ++i) {((uint8_t*)ptr)[i] = i + 1;}// 现在对刚刚的内存不感兴趣了，可以暂存的交换区VirtualAlloc(ptr, 0, MEM_RESET, PAGE_NOACCESS);VirtualQuery(ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);// 去申请下一块内存size_t next_expect = expect - actual;void* next_ptr = (void*)((size_t)ptr + actual);const size_t next_actual = MyCommit(next_ptr, next_expect);VirtualQuery(next_ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);assert(next_actual);for (int i = 0; i < 10; ++i) {((uint8_t*)next_ptr)[i] = i + 1;}// 把申请到的next_ptr 暂存到交换区VirtualAlloc(next_ptr, 0, MEM_RESET, PAGE_NOACCESS);VirtualQuery(next_ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);// 把ptr处的内存从交换区中取出VirtualAlloc(ptr, actual, MEM_RESET_UNDO, PAGE_READWRITE);VirtualQuery(ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);// 读取数据for (int i = 0; i < 10; ++i) {std::cout << int(((uint8_t*)ptr)[i]) << std::endl;}VirtualFree(ptr, 0, MEM_RELEASE);VirtualQuery(ptr, &memInfo, sizeof(memInfo));printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);return EXIT_SUCCESS;
}

如果是32位的应用程序像突破4GB的寻址限制，需要使用到地址窗口化扩展， 简称AWE。
微软官方示例

#include "stdafx.h"
#include <windows.h>
#include <stdio.h>
#include <tchar.h>constexpr size_t MEMORY_REQUESTED = 1024 * 1024;BOOL
LoggedSetLockPagesPrivilege(HANDLE hProcess,BOOL bEnable);size_t GetPageSize() {static size_t page_size = 0;if (page_size == 0) {SYSTEM_INFO sSysInfo;GetSystemInfo(&sSysInfo);page_size = sSysInfo.dwPageSize;}return page_size;
}void  main()
{BOOL bResult;                   // generic Boolean valueULONG_PTR NumberOfPages;        // number of pages to requestULONG_PTR NumberOfPagesInitial; // initial number of pages requestedULONG_PTR *aPFNs;               // page info; holds opaque dataPVOID lpMemReserved;            // AWE windowint PFNArraySize;               // memory to request for PFN array_tprintf(_T("This computer has page size %d.\n"), GetPageSize());// Calculate the number of pages of memory to request.NumberOfPages = MEMORY_REQUESTED / GetPageSize();_tprintf(_T("Requesting %d pages of memory.\n"), NumberOfPages);// Calculate the size of the user PFN array.PFNArraySize = NumberOfPages * sizeof(ULONG_PTR);_tprintf(_T("Requesting a PFN array of %d bytes.\n"), PFNArraySize);aPFNs = (ULONG_PTR *)HeapAlloc(GetProcessHeap(), 0, PFNArraySize);if (aPFNs == NULL){_tprintf(_T("Failed to allocate on heap.\n"));return;}// Enable the privilege.if (!LoggedSetLockPagesPrivilege(GetCurrentProcess(), TRUE)){return;}// Allocate the physical memory.NumberOfPagesInitial = NumberOfPages;bResult = AllocateUserPhysicalPages(GetCurrentProcess(),&NumberOfPages,aPFNs);if (bResult != TRUE){_tprintf(_T("Cannot allocate physical pages (%u)\n"), GetLastError());return;}if (NumberOfPagesInitial != NumberOfPages){_tprintf(_T("Allocated only %p pages.\n"), NumberOfPages);return;}// Reserve the virtual memory.lpMemReserved = VirtualAlloc(NULL,MEMORY_REQUESTED,MEM_RESERVE | MEM_PHYSICAL,PAGE_READWRITE);if (lpMemReserved == NULL){_tprintf(_T("Cannot reserve memory.\n"));return;}// Map the physical memory into the window.bResult = MapUserPhysicalPages(lpMemReserved,NumberOfPages - 3,&aPFNs[3]);if (bResult != TRUE){_tprintf(_T("MapUserPhysicalPages failed (%u)\n"), GetLastError());return;}// unmapbResult = MapUserPhysicalPages(lpMemReserved,NumberOfPages - 3,NULL);if (bResult != TRUE){_tprintf(_T("MapUserPhysicalPages failed (%u)\n"), GetLastError());return;}// Free the physical pages.bResult = FreeUserPhysicalPages(GetCurrentProcess(),&NumberOfPages,aPFNs);if (bResult != TRUE){_tprintf(_T("Cannot free physical pages, error %u.\n"), GetLastError());return;}// Free virtual memory.bResult = VirtualFree(lpMemReserved,0,MEM_RELEASE);// Release the aPFNs array.bResult = HeapFree(GetProcessHeap(), 0, aPFNs);if (bResult != TRUE){_tprintf(_T("Call to HeapFree has failed (%u)\n"), GetLastError());}}/*****************************************************************
LoggedSetLockPagesPrivilege: a function to obtain or
release the privilege of locking physical pages.Inputs:HANDLE hProcess: Handle for the process for which the
privilege is neededBOOL bEnable: Enable (TRUE) or disable?Return value: TRUE indicates success, FALSE failure.*****************************************************************/
BOOL
LoggedSetLockPagesPrivilege(HANDLE hProcess,BOOL bEnable)
{struct {DWORD Count;LUID_AND_ATTRIBUTES Privilege[1];} Info;HANDLE Token;BOOL Result;// Open the token.Result = OpenProcessToken(hProcess,TOKEN_ADJUST_PRIVILEGES,&Token);if (Result != TRUE){_tprintf(_T("Cannot open process token.\n"));return FALSE;}// Enable or disable?Info.Count = 1;if (bEnable){Info.Privilege[0].Attributes = SE_PRIVILEGE_ENABLED;}else{Info.Privilege[0].Attributes = 0;}// Get the LUID.Result = LookupPrivilegeValue(NULL,SE_LOCK_MEMORY_NAME,&(Info.Privilege[0].Luid));if (Result != TRUE){_tprintf(_T("Cannot get privilege for %s.\n"), SE_LOCK_MEMORY_NAME);return FALSE;}// Adjust the privilege.Result = AdjustTokenPrivileges(Token, FALSE,(PTOKEN_PRIVILEGES)&Info,0, NULL, NULL);// Check the result.if (Result != TRUE){_tprintf(_T("Cannot adjust token privileges (%u)\n"), GetLastError());return FALSE;}else{if (GetLastError() != ERROR_SUCCESS){_tprintf(_T("Cannot enable the SE_LOCK_MEMORY_NAME privilege; "));_tprintf(_T("please check the local policy.\n"));return FALSE;}}CloseHandle(Token);return TRUE;
}