当前位置： 首页 > news >正文

使用内存映射读取文件和写入文件，并进行性能测试

news 2025/9/18 13:54:48

文章目录

    • 实现文件
    • 主文件进行测试
    • 测试结果
  • 内存映射文件 vs 传统文件I/O性能测试结果分析
    • 1. 测试结果概览
    • 2. 详细分析
      • 2.1 读取性能
        • 顺序读取：
        • 随机读取：
      • 2.2 写入性能
        • 顺序写入：
        • 随机写入：
      • 2.3 特殊现象分析
    • 3. 综合结论与建议
      • 3.1 何时使用内存映射
      • 3.2 何时使用传统fstream
      • 3.3 最佳实践建议
    • 4. 进一步测试建议

实现文件

#pragma once
#include <filesystem>
#include <vector>
#include <memory>
#include <stdexcept>
#include <functional>
#include <cstring>#if defined(_WIN32)
#include <windows.h>
#else
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#endif
namespace fs = std::filesystem;
class MemoryMappedFile {
public:// 构造时立即映射文件（可选拷贝）explicit MemoryMappedFile(const fs::path& file_path, bool force_copy = false) {Open(file_path, force_copy);}// 默认构造（延迟打开）MemoryMappedFile() = default;// 支持拷贝语义（通过共享指针实现安全共享）MemoryMappedFile(const MemoryMappedFile& other) = default;MemoryMappedFile& operator=(const MemoryMappedFile& other) = default;// 支持移动语义MemoryMappedFile(MemoryMappedFile&& other) noexcept = default;MemoryMappedFile& operator=(MemoryMappedFile&& other) noexcept = default;~MemoryMappedFile() = default;// 打开/重新打开文件bool Open(const fs::path& file_path, bool force_copy = false) {Close();if (!fs::exists(file_path)) {throw std::runtime_error("File does not exist: " + file_path.string());}size_ = fs::file_size(file_path);if (size_ == 0) {return true;}if (force_copy) {// 直接读取文件到内存，避免先映射再拷贝的开销ReadFileToMemory(file_path);is_mapped_ = false;return true;}// 使用内存映射
#if defined(_WIN32)return MapFileWindows(file_path);
#elsereturn MapFileUnix(file_path);
#endif}// 关闭文件映射void Close() {view_.reset();
#if defined(_WIN32)hMapping_.reset();hFile_.reset();
#elseif (fd_) {close(*fd_);fd_.reset();}
#endifdata_.reset();size_ = 0;is_mapped_ = false;}// 访问数据const uint8_t* data() const {return is_mapped_ ?static_cast<const uint8_t*>(view_.get()) :(data_ ? data_->data() : nullptr);}size_t size() const { return size_; }bool is_mapped() const { return is_mapped_; }bool is_open() const { return size_ > 0; }private:size_t size_ = 0;bool is_mapped_ = false;// 使用共享指针管理资源
#if defined(_WIN32)std::shared_ptr<void> hFile_;std::shared_ptr<void> hMapping_;
#elsestd::shared_ptr<int> fd_; // 包装文件描述符
#endifstd::shared_ptr<void> view_; // 映射视图std::shared_ptr<std::vector<uint8_t>> data_; // 数据副本// 读取文件到内存（用于force_copy情况）void ReadFileToMemory(const fs::path& file_path) {std::ifstream file(file_path, std::ios::binary);if (!file) {throw std::runtime_error("Failed to open file for reading: " + file_path.string());}data_ = std::make_shared<std::vector<uint8_t>>(size_);file.read(reinterpret_cast<char*>(data_->data()), size_);if (!file) {data_.reset();throw std::runtime_error("Failed to read file: " + file_path.string());}}#if defined(_WIN32)// Windows文件映射bool MapFileWindows(const fs::path& file_path) {// Windows资源获取HANDLE hFile = CreateFileW(file_path.c_str(),GENERIC_READ,FILE_SHARE_READ,nullptr,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,nullptr);if (hFile == INVALID_HANDLE_VALUE) {throw std::runtime_error("Failed to open file: " + GetLastErrorString());}hFile_ = std::shared_ptr<void>(hFile, [](void* handle) {CloseHandle(static_cast<HANDLE>(handle));});HANDLE hMapping = CreateFileMappingW(hFile,nullptr,PAGE_READONLY,0, 0,nullptr);if (!hMapping) {throw std::runtime_error("Failed to create file mapping: " + GetLastErrorString());}hMapping_ = std::shared_ptr<void>(hMapping, [](void* handle) {CloseHandle(static_cast<HANDLE>(handle));});// 使用自定义删除器管理映射视图void* view = MapViewOfFile(hMapping,FILE_MAP_READ,0, 0,size_);if (!view) {throw std::runtime_error("Failed to map view of file: " + GetLastErrorString());}view_ = std::shared_ptr<void>(view, [](void* view) {UnmapViewOfFile(view);});is_mapped_ = true;return true;}// 获取Windows错误信息static std::string GetLastErrorString() {DWORD error = GetLastError();if (error == 0) {return "";}LPSTR buffer = nullptr;size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,nullptr, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),reinterpret_cast<LPSTR>(&buffer), 0, nullptr);std::string message(buffer, size);LocalFree(buffer);return message;}
#else// Unix文件映射bool MapFileUnix(const fs::path& file_path) {int fd = open(file_path.c_str(), O_RDONLY);if (fd == -1) {throw std::runtime_error("Failed to open file: " + GetErrnoString());}fd_ = std::shared_ptr<int>(new int(fd), [](int* fd) {if (*fd != -1) {close(*fd);}delete fd;});void* view = mmap(nullptr,size_,PROT_READ,MAP_PRIVATE,fd,0);if (view == MAP_FAILED) {throw std::runtime_error("Failed to map file: " + GetErrnoString());}// 使用自定义删除器管理映射view_ = std::shared_ptr<void>(view, [size = size_](void* view) {munmap(view, size);});is_mapped_ = true;return true;}// 获取Unix错误信息static std::string GetErrnoString() {return std::strerror(errno);}
#endif
};#if 0//#include <iostream>
//#include "MemoryMappedFile.h" // 假设类定义在此头文件中int main() {try {// 示例 1: 使用内存映射方式打开文件std::cout << "示例 1: 使用内存映射方式打开文件" << std::endl;MemoryMappedFile mappedFile("example.txt");if (mappedFile.is_open()) {std::cout << "文件大小: " << mappedFile.size() << " 字节" << std::endl;std::cout << "是否使用内存映射: " << (mappedFile.is_mapped() ? "是" : "否") << std::endl;// 访问文件内容const uint8_t* data = mappedFile.data();// 处理文件内容...std::cout << "前10字节: ";for (size_t i = 0; i < std::min(static_cast<size_t>(10), mappedFile.size()); ++i) {std::cout << std::hex << static_cast<int>(data[i]) << " ";}std::cout << std::endl;}// 示例 2: 强制拷贝方式打开文件std::cout << "\n示例 2: 强制拷贝方式打开文件" << std::endl;MemoryMappedFile copiedFile("example.txt", true);if (copiedFile.is_open()) {std::cout << "文件大小: " << copiedFile.size() << " 字节" << std::endl;std::cout << "是否使用内存映射: " << (copiedFile.is_mapped() ? "是" : "否") << std::endl;// 访问文件内容const uint8_t* data = copiedFile.data();// 处理文件内容...}// 示例 3: 延迟打开文件std::cout << "\n示例 3: 延迟打开文件" << std::endl;MemoryMappedFile delayedFile;// 稍后打开文件delayedFile.Open("example.txt");if (delayedFile.is_open()) {std::cout << "文件已成功打开" << std::endl;}// 示例 4: 处理不存在的文件std::cout << "\n示例 4: 处理不存在的文件" << std::endl;try {MemoryMappedFile nonexistentFile("nonexistent.txt");}catch (const std::exception& e) {std::cout << "错误: " << e.what() << std::endl;}// 示例 5: 复制和移动语义std::cout << "\n示例 5: 复制和移动语义" << std::endl;MemoryMappedFile original("example.txt");MemoryMappedFile copy = original; // 复制构造MemoryMappedFile moved = std::move(original); // 移动构造}catch (const std::exception& e) {std::cerr << "发生错误: " << e.what() << std::endl;return 1;}return 0;
}
#endif#pragma once
#include <filesystem>
#include <vector>
#include <memory>
#include <stdexcept>
#include <functional>
#include <cstring>
#include <fstream>#if defined(_WIN32)
#include <windows.h>
#else
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#endif//namespace fs = std::filesystem;class MemoryMappedFileWriter {
public:// 构造时立即创建或打开文件进行映射explicit MemoryMappedFileWriter(const fs::path& file_path, size_t size = 0, bool truncate = true) {Open(file_path, size, truncate);}// 默认构造（延迟打开）MemoryMappedFileWriter() = default;// 禁止拷贝（因为涉及独占资源）MemoryMappedFileWriter(const MemoryMappedFileWriter&) = delete;MemoryMappedFileWriter& operator=(const MemoryMappedFileWriter&) = delete;// 支持移动语义MemoryMappedFileWriter(MemoryMappedFileWriter&& other) noexcept = default;MemoryMappedFileWriter& operator=(MemoryMappedFileWriter&& other) noexcept = default;~MemoryMappedFileWriter() {Close();}// 打开/重新打开文件bool Open(const fs::path& file_path, size_t size = 0, bool truncate = true) {Close();file_path_ = file_path;size_ = size;#if defined(_WIN32)return MapFileWindows(file_path, size, truncate);
#elsereturn MapFileUnix(file_path, size, truncate);
#endif}// 关闭文件映射void Close() {// 先刷新数据Flush();// 释放资源
#if defined(_WIN32)if (view_) {UnmapViewOfFile(view_.get());view_.reset();}if (hMapping_) {CloseHandle(hMapping_.get());hMapping_.reset();}if (hFile_) {CloseHandle(hFile_.get());hFile_.reset();}
#elseif (view_) {munmap(view_.get(), size_);view_.reset();}if (fd_) {close(*fd_);fd_.reset();}
#endifsize_ = 0;is_mapped_ = false;}// 刷新数据到磁盘bool Flush() {if (!is_mapped_) return false;#if defined(_WIN32)return FlushViewOfFile(view_.get(), size_) != FALSE;
#elsereturn msync(view_.get(), size_, MS_SYNC) == 0;
#endif}// 调整文件大小（需要重新映射）bool Resize(size_t new_size) {if (!is_mapped_) return false;// 保存当前数据std::vector<uint8_t> temp_data(size_);if (size_ > 0) {memcpy(temp_data.data(), view_.get(), size_);}// 关闭当前映射Close();// 重新打开并映射return Open(file_path_, new_size, true) &&(new_size <= size_ || memcpy(view_.get(), temp_data.data(), size_) != nullptr);}// 访问数据uint8_t* data() { return static_cast<uint8_t*>(view_.get()); }const uint8_t* data() const { return static_cast<const uint8_t*>(view_.get()); }size_t size() const { return size_; }bool is_mapped() const { return is_mapped_; }bool is_open() const { return is_mapped_; }private:fs::path file_path_;size_t size_ = 0;bool is_mapped_ = false;// 使用唯一指针管理资源
#if defined(_WIN32)struct HandleDeleter {void operator()(void* handle) const {if (handle != INVALID_HANDLE_VALUE && handle != nullptr) {CloseHandle(static_cast<HANDLE>(handle));}}};std::unique_ptr<void, HandleDeleter> hFile_;std::unique_ptr<void, HandleDeleter> hMapping_;
#elsestruct FdDeleter {void operator()(int* fd) const {if (fd && *fd != -1) {close(*fd);delete fd;}}};std::unique_ptr<int, FdDeleter> fd_;
#endif// 修改 ViewDeleter 以支持默认构造struct ViewDeleter {size_t size;// 添加默认构造函数ViewDeleter() : size(0) {}ViewDeleter(size_t s) : size(s) {}void operator()(void* view) const {if (view) {
#if defined(_WIN32)UnmapViewOfFile(view);
#elsemunmap(view, size);
#endif}}};// 使用 shared_ptr 而不是 unique_ptr 来避免默认构造问题std::shared_ptr<void> view_;#if defined(_WIN32)// Windows文件映射bool MapFileWindows(const fs::path& file_path, size_t size, bool truncate) {// 创建或打开文件HANDLE hFile = CreateFileW(file_path.c_str(),GENERIC_READ | GENERIC_WRITE,FILE_SHARE_READ,nullptr,truncate ? CREATE_ALWAYS : OPEN_ALWAYS,FILE_ATTRIBUTE_NORMAL,nullptr);if (hFile == INVALID_HANDLE_VALUE) {throw std::runtime_error("Failed to open file: " + GetLastErrorString());}hFile_.reset(hFile);// 如果需要，调整文件大小if (size > 0) {LARGE_INTEGER liSize;liSize.QuadPart = size;if (SetFilePointerEx(hFile, liSize, nullptr, FILE_BEGIN) == FALSE ||SetEndOfFile(hFile) == FALSE) {throw std::runtime_error("Failed to set file size: " + GetLastErrorString());}size_ = size;}else {// 获取当前文件大小LARGE_INTEGER fileSize;if (!GetFileSizeEx(hFile, &fileSize)) {throw std::runtime_error("Failed to get file size: " + GetLastErrorString());}size_ = fileSize.QuadPart;}if (size_ == 0) {// 空文件不需要映射return true;}// 创建文件映射HANDLE hMapping = CreateFileMappingW(hFile,nullptr,PAGE_READWRITE,0, 0,nullptr);if (!hMapping) {throw std::runtime_error("Failed to create file mapping: " + GetLastErrorString());}hMapping_.reset(hMapping);// 映射视图void* view = MapViewOfFile(hMapping,FILE_MAP_WRITE,0, 0,size_);if (!view) {throw std::runtime_error("Failed to map view of file: " + GetLastErrorString());}// 使用自定义删除器创建 shared_ptrview_ = std::shared_ptr<void>(view, ViewDeleter(size_));is_mapped_ = true;return true;}// 获取Windows错误信息static std::string GetLastErrorString() {DWORD error = GetLastError();if (error == 0) {return "";}LPSTR buffer = nullptr;size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,nullptr, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),reinterpret_cast<LPSTR>(&buffer), 0, nullptr);std::string message(buffer, size);LocalFree(buffer);return message;}
#else// Unix文件映射bool MapFileUnix(const fs::path& file_path, size_t size, bool truncate) {// 创建或打开文件int fd = open(file_path.c_str(), O_RDWR | O_CREAT | (truncate ? O_TRUNC : 0), 0644);if (fd == -1) {throw std::runtime_error("Failed to open file: " + GetErrnoString());}fd_.reset(new int(fd));// 如果需要，调整文件大小if (size > 0) {if (ftruncate(fd, size) == -1) {throw std::runtime_error("Failed to set file size: " + GetErrnoString());}size_ = size;}else {// 获取当前文件大小struct stat st;if (fstat(fd, &st) == -1) {throw std::runtime_error("Failed to get file size: " + GetErrnoString());}size_ = st.st_size;}if (size_ == 0) {// 空文件不需要映射return true;}// 映射文件void* view = mmap(nullptr,size_,PROT_READ | PROT_WRITE,MAP_SHARED,fd,0);if (view == MAP_FAILED) {throw std::runtime_error("Failed to map file: " + GetErrnoString());}// 使用自定义删除器创建 shared_ptrview_ = std::shared_ptr<void>(view, ViewDeleter(size_));is_mapped_ = true;return true;}// 获取Unix错误信息static std::string GetErrnoString() {return std::strerror(errno);}
#endif
};#if 0#include <iostream>
#include <vector>
#include <cstring>
#include <filesystem>
#include "MemoryMappedFileWriter.h"namespace fs = std::filesystem;// 生成测试数据
std::vector<uint8_t> GenerateTestData(size_t size) {std::vector<uint8_t> data(size);for (size_t i = 0; i < size; ++i) {data[i] = static_cast<uint8_t>(i % 256);}return data;
}// 验证文件内容
bool VerifyFileContent(const fs::path& file_path, const std::vector<uint8_t>& expected) {std::ifstream file(file_path, std::ios::binary);if (!file) {std::cerr << "Failed to open file for verification: " << file_path << std::endl;return false;}// 获取文件大小file.seekg(0, std::ios::end);size_t file_size = file.tellg();file.seekg(0, std::ios::beg);if (file_size != expected.size()) {std::cerr << "File size mismatch: expected " << expected.size()<< ", actual " << file_size << std::endl;return false;}// 读取文件内容std::vector<uint8_t> actual(file_size);file.read(reinterpret_cast<char*>(actual.data()), file_size);// 比较内容if (memcmp(actual.data(), expected.data(), file_size) != 0) {std::cerr << "File content mismatch" << std::endl;return false;}return true;
}int main() {const fs::path test_file = "memory_mapped_test.bin";const size_t initial_size = 1024 * 1024; // 1MBconst size_t resized_size = 2 * 1024 * 1024; // 2MBtry {// === 示例1: 创建新文件并写入数据 ===std::cout << "=== Creating new file ===" << std::endl;{// 生成测试数据auto test_data = GenerateTestData(initial_size);// 创建内存映射文件写入器MemoryMappedFileWriter writer(test_file, initial_size);if (!writer.is_mapped()) {std::cerr << "Failed to map file" << std::endl;return 1;}std::cout << "File mapped successfully. Size: "<< writer.size() << " bytes" << std::endl;// 将测试数据复制到映射内存memcpy(writer.data(), test_data.data(), initial_size);// 显式刷新数据到磁盘if (!writer.Flush()) {std::cerr << "Failed to flush data" << std::endl;}// 验证文件内容if (VerifyFileContent(test_file, test_data)) {std::cout << "File creation verified successfully" << std::endl;}} // writer 在此作用域结束时自动关闭// === 示例2: 修改现有文件 ===std::cout << "\n=== Modifying existing file ===" << std::endl;{// 打开现有文件MemoryMappedFileWriter writer(test_file, 0, false);if (!writer.is_mapped()) {std::cerr << "Failed to map existing file" << std::endl;return 1;}std::cout << "Existing file mapped. Size: "<< writer.size() << " bytes" << std::endl;// 修改文件内容 - 在文件开头添加标记const char* header = "MODIFIED";memcpy(writer.data(), header, strlen(header));// 在文件中间添加标记size_t mid_point = writer.size() / 2;const char* mid_marker = "MIDDLE";memcpy(writer.data() + mid_point, mid_marker, strlen(mid_marker));// 不需要显式刷新，析构时会自动刷新}// 验证修改后的文件{// 生成预期的修改后数据auto expected_data = GenerateTestData(initial_size);memcpy(expected_data.data(), "MODIFIED", 8);memcpy(expected_data.data() + initial_size / 2, "MIDDLE", 6);if (VerifyFileContent(test_file, expected_data)) {std::cout << "File modification verified successfully" << std::endl;}}// === 示例3: 调整文件大小 ===std::cout << "\n=== Resizing file ===" << std::endl;{// 打开现有文件MemoryMappedFileWriter writer(test_file, 0, false);if (!writer.is_mapped()) {std::cerr << "Failed to map file for resizing" << std::endl;return 1;}std::cout << "Original size: " << writer.size() << " bytes" << std::endl;// 调整文件大小if (!writer.Resize(resized_size)) {std::cerr << "Failed to resize file" << std::endl;return 1;}std::cout << "New size: " << writer.size() << " bytes" << std::endl;// 初始化新增加的空间size_t original_size = initial_size;size_t new_space = resized_size - original_size;memset(writer.data() + original_size, 0xFF, new_space);// 在文件末尾添加标记const char* footer = "RESIZED";memcpy(writer.data() + resized_size - strlen(footer), footer, strlen(footer));}// 验证调整大小后的文件{// 生成预期的调整后数据auto expected_data = GenerateTestData(initial_size);memcpy(expected_data.data(), "MODIFIED", 8);memcpy(expected_data.data() + initial_size / 2, "MIDDLE", 6);// 扩展数据到新大小expected_data.resize(resized_size, 0xFF);// 设置文件末尾标记const char* footer = "RESIZED";memcpy(expected_data.data() + resized_size - strlen(footer), footer, strlen(footer));if (VerifyFileContent(test_file, expected_data)) {std::cout << "File resize verified successfully" << std::endl;}}// === 示例4: 使用移动语义 ===std::cout << "\n=== Using move semantics ===" << std::endl;{// 创建临时写入器MemoryMappedFileWriter temp_writer("temp_file.bin", 512);memcpy(temp_writer.data(), "Temporary data", 14);// 移动构造MemoryMappedFileWriter moved_writer(std::move(temp_writer));// 验证移动后的对象if (moved_writer.is_mapped()) {std::cout << "Moved writer is mapped. Data: "<< reinterpret_cast<const char*>(moved_writer.data()) << std::endl;}// 移动赋值MemoryMappedFileWriter another_writer;another_writer = std::move(moved_writer);// 验证再次移动后的对象if (another_writer.is_mapped()) {std::cout << "Reassigned writer is mapped. Data: "<< reinterpret_cast<const char*>(another_writer.data()) << std::endl;}}std::cout << "\nAll operations completed successfully!" << std::endl;}catch (const std::exception& e) {std::cerr << "Error: " << e.what() << std::endl;return 1;}// 清理测试文件try {if (fs::exists(test_file)) {fs::remove(test_file);}if (fs::exists("temp_file.bin")) {fs::remove("temp_file.bin");}}catch (...) {// 忽略清理错误}return 0;
}#endif

主文件进行测试

#include <chrono>
#include <iostream>
#include <fstream>
#include <vector>
#include "MemoryMappedFile.hpp"
#include <random>
#include <numeric>
#include <string>
//#include "MemoryMappedFileWriter.h"namespace fs = std::filesystem;
using namespace std::chrono;// 测试文件大小配置
const size_t KB = 1024;
const size_t MB = KB * KB;
const size_t GB = MB * KB;
const std::vector<size_t> TEST_SIZES = { 4 * KB, 64 * KB, 1 * MB, 16 * MB, 128 * MB, 1 * GB };void TestSequentialRead(const fs::path& file_path) {size_t file_size = fs::file_size(file_path);std::vector<uint8_t> buffer(file_size);// 测试ifstreamauto start = high_resolution_clock::now();{std::ifstream file(file_path, std::ios::binary);file.read(reinterpret_cast<char*>(buffer.data()), file_size);}auto ifstream_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();// 测试内存映射文件start = high_resolution_clock::now();{MemoryMappedFile mapped_file(file_path);memcpy(buffer.data(), mapped_file.data(), file_size);}auto mmap_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();std::cout << "Sequential Read (" << file_size / MB << "MB): "<< "ifstream=" << ifstream_time << "ms, "<< "mmap=" << mmap_time << "ms" << std::endl;
}
void TestRandomRead(const fs::path& file_path) {size_t file_size = fs::file_size(file_path);const int ACCESS_COUNT = 10000;std::vector<size_t> access_points(ACCESS_COUNT);// 生成随机访问点 - 使用合适的整数类型std::random_device rd;std::mt19937 gen(rd());// 将size_t转换为unsigned long longstd::uniform_int_distribution<unsigned long long> dist(0, file_size - 1);for (auto& pos : access_points) {pos = static_cast<size_t>(dist(gen));}uint8_t temp;// 测试ifstreamauto start = high_resolution_clock::now();{std::ifstream file(file_path, std::ios::binary);for (auto pos : access_points) {file.seekg(pos);file.read(reinterpret_cast<char*>(&temp), 1);}}auto ifstream_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();// 测试内存映射文件start = high_resolution_clock::now();{MemoryMappedFile mapped_file(file_path);for (auto pos : access_points) {temp = mapped_file.data()[pos];}}auto mmap_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();std::cout << "Random Read (" << ACCESS_COUNT << " accesses): "<< "ifstream=" << ifstream_time << "ms, "<< "mmap=" << mmap_time << "ms" << std::endl;
}void TestSequentialWrite(size_t size) {fs::path file1 = "D:/fstream_write_test.bin";fs::path file2 = "D:/mmap_write_test.bin";std::vector<uint8_t> data(size);std::iota(data.begin(), data.end(), 0); // 填充测试数据// 测试ofstreamauto start = high_resolution_clock::now();{std::ofstream file(file1, std::ios::binary);file.write(reinterpret_cast<const char*>(data.data()), size);}auto fstream_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();// 测试内存映射写入start = high_resolution_clock::now();{MemoryMappedFileWriter writer(file2, size);memcpy(writer.data(), data.data(), size);}auto mmap_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();std::cout << "Sequential Write (" << size / MB << "MB): "<< "ofstream=" << fstream_time << "ms, "<< "mmap=" << mmap_time << "ms" << std::endl;// 清理测试文件fs::remove(file1);fs::remove(file2);
}
void TestRandomWrite(size_t size) {fs::path file1 = "D:/fstream_random_write_test.bin";fs::path file2 = "D:/mmap_random_write_test.bin";const int WRITE_COUNT = 10000;// 生成随机写入位置和数据 - 使用合适的整数类型std::random_device rd;std::mt19937 gen(rd());// 将size_t转换为unsigned long longstd::uniform_int_distribution<unsigned long long> pos_dist(0, size - 1);std::uniform_int_distribution<uint16_t> data_dist(0, 255); // uint8_t也有效// 测试ofstreamauto start = high_resolution_clock::now();{std::ofstream file(file1, std::ios::binary);// 先填充文件std::vector<uint8_t> init_data(size, 0);file.write(reinterpret_cast<const char*>(init_data.data()), size);// 随机写入for (int i = 0; i < WRITE_COUNT; ++i) {size_t pos = static_cast<size_t>(pos_dist(gen));uint8_t value = static_cast<uint8_t>(data_dist(gen));file.seekp(pos);file.write(reinterpret_cast<const char*>(&value), 1);}}auto fstream_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();// 测试内存映射写入start = high_resolution_clock::now();{MemoryMappedFileWriter writer(file2, size);// 初始化文件memset(writer.data(), 0, size);// 随机写入for (int i = 0; i < WRITE_COUNT; ++i) {size_t pos = pos_dist(gen);uint8_t value = data_dist(gen);writer.data()[pos] = value;}}auto mmap_time = duration_cast<milliseconds>(high_resolution_clock::now() - start).count();std::cout << "Random Write (" << WRITE_COUNT << " writes): "<< "ofstream=" << fstream_time << "ms, "<< "mmap=" << mmap_time << "ms" << std::endl;// 清理测试文件fs::remove(file1);fs::remove(file2);
}int main() {// 准备测试文件for (auto size : TEST_SIZES) {fs::path test_file = "test_" + std::to_string(size / MB) + "MB.bin";// 生成测试文件if (!fs::exists(test_file)) {std::cout << "Generating test file: " << test_file << std::endl;std::vector<uint8_t> data(size);std::iota(data.begin(), data.end(), 0);std::ofstream file(test_file, std::ios::binary);file.write(reinterpret_cast<const char*>(data.data()), size);}// 执行测试std::cout << "\n=== Testing with " << size / MB << "MB file ===" << std::endl;TestSequentialRead(test_file);TestRandomRead(test_file);TestSequentialWrite(size);TestRandomWrite(size);// 清理测试文件fs::remove(test_file);}return 0;
}

测试结果

Generating test file: "test_0MB.bin"=== Testing with 0MB file ===
Sequential Read (0MB): ifstream=0ms, mmap=0ms
Random Read (10000 accesses): ifstream=29ms, mmap=1ms
Sequential Write (0MB): ofstream=0ms, mmap=0ms
Random Write (10000 writes): ofstream=144ms, mmap=2ms
Generating test file: "test_0MB.bin"=== Testing with 0MB file ===
Sequential Read (0MB): ifstream=0ms, mmap=0ms
Random Read (10000 accesses): ifstream=33ms, mmap=0ms
Sequential Write (0MB): ofstream=0ms, mmap=1ms
Random Write (10000 writes): ofstream=65ms, mmap=3ms
Generating test file: "test_1MB.bin"=== Testing with 1MB file ===
Sequential Read (1MB): ifstream=1ms, mmap=0ms
Random Read (10000 accesses): ifstream=33ms, mmap=0ms
Sequential Write (1MB): ofstream=1ms, mmap=3ms
Random Write (10000 writes): ofstream=68ms, mmap=5ms
Generating test file: "test_16MB.bin"=== Testing with 16MB file ===
Sequential Read (16MB): ifstream=13ms, mmap=5ms
Random Read (10000 accesses): ifstream=47ms, mmap=3ms
Sequential Write (16MB): ofstream=4ms, mmap=41ms
Random Write (10000 writes): ofstream=91ms, mmap=42ms
Generating test file: "test_128MB.bin"=== Testing with 128MB file ===
Sequential Read (128MB): ifstream=107ms, mmap=53ms
Random Read (10000 accesses): ifstream=54ms, mmap=11ms
Sequential Write (128MB): ofstream=282ms, mmap=314ms
Random Write (10000 writes): ofstream=432ms, mmap=321ms=== Testing with 1024MB file ===
Sequential Read (1024MB): ifstream=895ms, mmap=416ms
Random Read (10000 accesses): ifstream=57ms, mmap=15ms
Sequential Write (1024MB): ofstream=2631ms, mmap=2622ms
Random Write (10000 writes): ofstream=3264ms, mmap=2547ms

内存映射文件 vs 传统文件I/O性能测试结果分析

根据您提供的测试结果，我将从不同角度对内存映射文件(MemoryMappedFile)与传统fstream/ifstream的性能进行详细分析。

1. 测试结果概览

文件大小测试类型ifstream/ofstream(ms)内存映射(ms)性能比(传统/映射)
0MB顺序读取00-
0MB随机读取29-330-129-33x
0MB顺序写入00-1-
0MB随机写入65-1442-321.6-48x
1MB顺序读取10∞
1MB随机读取330∞
1MB顺序写入130.33x
1MB随机写入68513.6x
16MB顺序读取1352.6x
16MB随机读取47315.7x
16MB顺序写入4410.1x
16MB随机写入91422.2x
128MB顺序读取107532x
128MB随机读取54114.9x
128MB顺序写入2823140.9x
128MB随机写入4323211.3x
1024MB顺序读取8954162.2x
1024MB随机读取57153.8x
1024MB顺序写入263126221x
1024MB随机写入326425471.3x

2. 详细分析

2.1 读取性能

顺序读取：

  • ​小文件(0-1MB)​​：内存映射有轻微优势，但差异不大

  • ​中等文件(16-128MB)​​：内存映射快2-2.6倍

  • ​大文件(1GB)​​：内存映射快2.2倍

​结论​：文件越大，内存映射的顺序读取优势越明显，因为避免了用户空间和内核空间之间的数据拷贝。

随机读取：

  • ​所有文件大小​：内存映射显著更快(3.8-33倍)

  • ​特别优势​：在0MB文件测试中，内存映射几乎零开销(0-1ms vs 29-33ms)

​结论​：随机访问场景下，内存映射具有压倒性优势，因为它直接操作内存而无需频繁的seek操作。

2.2 写入性能

顺序写入：

  • ​小文件(1MB)​​：传统fstream更快(1ms vs 3ms)

  • ​中等文件(16MB)​​：传统fstream快10倍(4ms vs 41ms)

  • ​大文件(128MB-1GB)​​：两者性能相当

​异常分析​：16MB文件的顺序写入结果异常，内存映射反而更慢，可能是由于：

  1. 测试环境缓存影响

  2. 内存映射的初始化开销较大

  3. 写入时的同步机制差异

随机写入：

  • ​小文件(0-1MB)​​：内存映射快13.6-48倍

  • ​中等文件(16MB)​​：内存映射快2.2倍

  • ​大文件(128MB-1GB)​​：内存映射快1.3倍

​结论​：随机写入场景下，内存映射始终保持优势，特别是对小文件。

2.3 特殊现象分析

  1. ​0MB文件测试​：

    • 内存映射在随机访问中表现极佳，因为实际上没有数据需要处理

    • 传统I/O仍有显著开销，说明其基础操作成本较高

  2. ​16MB顺序写入异常​：

    • 可能是测试环境特有的现象

    • 建议重复测试确认是否为偶然现象

  3. ​大文件写入性能接近​：

    • 当文件足够大时，两种方法的性能趋于接近

    • 可能是因为I/O带宽成为瓶颈，而非方法本身

3. 综合结论与建议

3.1 何时使用内存映射

✅ ​强烈推荐使用内存映射的场景​：

  1. ​随机访问​（无论文件大小）

  2. ​大文件(>16MB)读取​

  3. ​需要频繁访问同一文件​

  4. ​多进程共享访问同一文件​

3.2 何时使用传统fstream

✅ ​传统fstream更合适的场景​：

  1. ​小文件(<1MB)顺序写入​

  2. ​需要精细控制I/O缓冲和刷新时机​

  3. ​内存受限环境（避免映射大文件）​​

  4. ​简单的顺序读写操作​

3.3 最佳实践建议

  1. ​混合使用策略​：

    // 对大文件随机访问使用内存映射
if (file_size > 16 * 1024 * 1024 || need_random_access) {MemoryMappedFile file(path);// 处理文件...
} 
// 对小文件顺序操作使用传统I/O
else {std::fstream file(path, std::ios::binary);// 处理文件...
}

  2. ​内存映射的优化技巧​：

    • 对于写入操作，适当控制Flush()调用频率

    • 考虑使用MAP_POPULATE(Linux)或PrefetchVirtualMemory(Windows)预加载文件

  3. ​传统I/O的优化技巧​：

    • 使用适当的缓冲区大小

    • 对于顺序访问，使用std::ios::sync_with_stdio(false)

    • 考虑使用内存缓冲+批量写入

4. 进一步测试建议

  1. ​增加测试重复次数​：消除偶然性影响

  2. ​测试不同文件系统​：对比HDD和SSD上的表现

  3. ​加入内存使用统计​：评估内存占用差异

  4. ​多线程访问测试​：评估并发性能

  5. ​真实工作负载测试​：使用实际应用场景的数据模式

这些测试结果表明，内存映射文件在大多数情况下（特别是随机访问和大文件处理）具有显著性能优势，但在特定场景下传统文件I/O可能更适合。根据具体应用需求选择合适的I/O方法至关重要。


文章转载自：

http://ZZZ0F6NW.fnpmf.cn
http://9oQ3B7wC.fnpmf.cn
http://FS8ckAse.fnpmf.cn
http://AYozlIFc.fnpmf.cn
http://bU0OUsc9.fnpmf.cn
http://9wfBH577.fnpmf.cn
http://PCB1KP4S.fnpmf.cn
http://gznHK4Cf.fnpmf.cn
http://M6ksT8yt.fnpmf.cn
http://yhpVbccU.fnpmf.cn
http://1Dmmo98i.fnpmf.cn
http://fEAVjcYu.fnpmf.cn
http://yZk9fwTz.fnpmf.cn
http://DOvJGDFY.fnpmf.cn
http://XeIZ9cts.fnpmf.cn
http://uTuPvXpu.fnpmf.cn
http://E1KsXGZk.fnpmf.cn
http://UJn6axr9.fnpmf.cn
http://mXXPpzlo.fnpmf.cn
http://4fReSm7Q.fnpmf.cn
http://8iS4PBJf.fnpmf.cn
http://4Y3mzkh9.fnpmf.cn
http://qXuabja2.fnpmf.cn
http://LyFmr0iY.fnpmf.cn
http://UslpNjnV.fnpmf.cn
http://42SIfBmz.fnpmf.cn
http://N982Eh8l.fnpmf.cn
http://3uOEWabT.fnpmf.cn
http://FiznOfoW.fnpmf.cn
http://mkOUE1Uw.fnpmf.cn
http://www.dtcms.com/a/388639.html

相关文章：

  • SQL的UNION用法大全介绍
  • 从Web原生到高性能：如何优化企业数据库管理工具
  • 基于python新能源汽车数据分析可视化系统 懂车帝 Scrapy爬虫 Django框架 Vue框架 大数据项目（源码+文档）✅
  • 线性回归和 softmax 回归
  • mysql远程访问连接设置
  • 《WINDOWS 环境下32位汇编语言程序设计》学习17章 PE文件（2）
  • Linux网络编程：从协议到实战
  • Vector 底层实现详解
  • OpenShift Virtualization - 虚机存储的相关概念 DataVolume、CDI 和 StorageProfile
  • 2025年Web自动化测试与Selenium面试题收集：从基础到进阶的全方位解析
  • pytorch中的FSDP
  • 贪心算法与材料切割问题详解
  • 2. 结构体
  • MySQL 核心操作：多表联合查询与数据库备份恢复
  • vue3学习日记(十四)：两大API选型指南
  • 微信支付回调成功通知到本地
  • 量化交易 - Simple Regression 简单线性回归（机器学习）
  • Kubernetes控制器详解：从Deployment到CronJob
  • python 架构技术50
  • 第九周文件上传
  • MCP大白话理解
  • 【Qt】QJsonValue存储 int64 类型的大整数时，数值出现莫名其妙的变化
  • 【C语言】冒泡排序算法解析与实现
  • [GESP202309 三级] 进制判断
  • 【C++】const和static的用法
  • 箭头函数{}规则，以及隐式返回
  • brain.js构建训练神经网络
  • 开学季高效学习与知识管理技术
  • C++STL与字符串探秘
  • 【面试题】- 使用CompletableFuture实现多线程统计策略工厂模式