MemoryMappedArray

class numpy::MemoryMappedArray

numpy C++ class.

Example

#include <numpy/np_ndarray.h>
using namespace numpy;

// Use MemoryMappedArray
MemoryMappedArray obj;
// ... operations ...

Constructors

Signature	Location	Example
MemoryMappedArray(const std::string &filename, const std::vector<size_t>&shape, MMapModemode = MMapMode::READ_WRITE, size_toffset = 0)	NP_MEMMAP.H:502
MemoryMappedArray(const MemoryMappedArray &)	NP_MEMMAP.H:565
noexcept MemoryMappedArray(MemoryMappedArray &&other)	NP_MEMMAP.H:569
MemoryMappedArray(const std::string &filename, boolread_only = true)	NP_MEMORY_POOL.H:323
MemoryMappedArray(const MemoryMappedArray &)	NP_MEMORY_POOL.H:327
noexcept MemoryMappedArray(MemoryMappedArray &&other)	NP_MEMORY_POOL.H:330

Operators

Signature	Return Type	Location	Example
MemoryMappedArray & operator=(const MemoryMappedArray &)	MemoryMappedArray &	NP_MEMMAP.H:566
MemoryMappedArray & noexcept operator=(MemoryMappedArray &&other)	MemoryMappedArray & noexcept	NP_MEMMAP.H:573
operator NDArray<T>&()		NP_MEMMAP.H:611
operator const NDArray<T>&()		NP_MEMMAP.H:612
MemoryMappedArray & operator=(const MemoryMappedArray &)	MemoryMappedArray &	NP_MEMORY_POOL.H:328
MemoryMappedArray & noexcept operator=(MemoryMappedArray &&other)	MemoryMappedArray & noexcept	NP_MEMORY_POOL.H:331
T & operator[](size_tindex)	T &	NP_MEMORY_POOL.H:337
const T & operator[](size_tindex)	const T &	NP_MEMORY_POOL.H:338

Construction

Signature	Return Type	Location	Example
MemoryMappedArray<T> from_npy(const std::string &filename, MMapModemode = MMapMode::READ_ONLY)	MemoryMappedArray<T>	NP_MEMMAP.H:541	View

Indexing / Selection

Signature	Return Type	Location	Example
T getElementAt(const std::vector<size_t>&indices)	T	NP_MEMMAP.H:622	View
const std::vector<size_t>& getShape()	const std::vector<size_t>&	NP_MEMMAP.H:619	View
size_t getSize()	size_t	NP_MEMMAP.H:620	View

Math Operations

Signature	Return Type	Location	Example
MMapMode mode()	MMapMode	NP_MEMMAP.H:603	View

Type Checking

Signature	Return Type	Location	Example
bool is_open()	bool	NP_MEMMAP.H:606	View
bool is_valid()	bool	NP_MEMORY_POOL.H:345

Other Methods

Signature	Return Type	Location	Example
NDArray<T>& array()	NDArray<T>&	NP_MEMMAP.H:615	View
const NDArray<T>& array()	const NDArray<T>&	NP_MEMMAP.H:616	View
T \* begin()	T *	NP_MEMORY_POOL.H:340	View
const T \* begin()	const T *	NP_MEMORY_POOL.H:342	View
void close()	void	NP_MEMMAP.H:592	View
T \* data()	T *	NP_MEMORY_POOL.H:333	View
const T \* data()	const T *	NP_MEMORY_POOL.H:334	View
T \* end()	T *	NP_MEMORY_POOL.H:341	View
const T \* end()	const T *	NP_MEMORY_POOL.H:343	View
const std::string & filename()	const std::string &	NP_MEMMAP.H:600
void flush()	void	NP_MEMMAP.H:584	View
void initializeMappedArray(const std::string &filename, const std::vector<size_t>&shape, MMapModemode, size_toffset)	void	NP_MEMMAP.H:522
void printArray()	void	NP_MEMMAP.H:628	View
void setElementAt(const std::vector<size_t>&indices, const T &value)	void	NP_MEMMAP.H:624	View
size_t size()	size_t	NP_MEMORY_POOL.H:335	View
std::string toString()	std::string	NP_MEMMAP.H:626	View

Code Examples

The following examples are extracted from the test suite.

from_npy (np_test_3_all.cpp:10846)

          for (size_t j = 0; j < 6; ++j) {
            arr.setElementAt({ i, j }, value++);
          }
        }
        save(arr, filename);
        // std::cout << "[OK] Created NPY file" << std::endl;
      }

      // Load as memory-mapped array
      {
        auto mmap_npy = memmap::MemoryMappedArray<double>::from_npy(filename, memmap::MMapMode::READ_WRITE);

        // Verify shape
        auto shape = mmap_npy.getShape();
        if (!(shape.size() == 2)) {
            std::string description = std::string("np_test_io_memmap_npy_files():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(shape.size() == 2)";
            std::cout << std::string("[FAIL] ") + description << std::endl;
            throw std::runtime_error(description);
        }
        if (!(shape[0] == 4)) {
            std::string description = std::string("np_test_io_memmap_npy_files():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(shape[0] == 4)";

getElementAt (np_test_1_all.cpp:49)

using namespace numpy::benchmark;
using namespace numpy::char_;

// Helper functions for array comparison tests
namespace {
    const double TOLERANCE = 1e-10;

    bool allTrue(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);
        do {
            if (!arr.getElementAt(indices)) {
                return false;
            }
        } while (incrementIndices(indices, arr.getShape()));
        return true;
    }

    bool allFalse(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);
        do {
            if (arr.getElementAt(indices)) {

getShape (np_test_1_all.cpp:47)

using namespace numpy;
using namespace numpy::benchmark;
using namespace numpy::char_;

// Helper functions for array comparison tests
namespace {
    const double TOLERANCE = 1e-10;

    bool allTrue(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);
        do {
            if (!arr.getElementAt(indices)) {
                return false;
            }
        } while (incrementIndices(indices, arr.getShape()));
        return true;
    }

    bool allFalse(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);

getSize (np_test_1_all.cpp:2172)

    }

    std::cout << " -> tests passed" << std::endl;
}

void testArrayEdgeCases() {
    std::cout << "========= testArrayEdgeCases =======================";

    // Test 1: Empty arrays
    NDArray<object_> empty_array = createObjectZeros({0});
    if (!(empty_array.getSize() == 0)) {
        std::string description = std::string("testArrayEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(empty_array.getSize() == 0)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }
    if (!(empty_array.getShape() == std::vector<size_t>{0})) {
        std::string description = std::string("testArrayEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(empty_array.getShape() == std::vector<size_t>{0})";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

mode (np_test_4_all.cpp:20711)

#include <stdexcept>

using namespace numpy;

namespace numpy_tests {
namespace numpy_tests_phase6b {

void np_test_phase6b_can_cast() {
    std::cout << "=== Test can_cast() ===";

    // Test SAFE casting mode (default)
    {
        // Same type is always safe
        if (!(can_cast(DType::INT32, DType::INT32, CastingMode::SAFE) == true)) {
            std::string description = std::string("np_test_phase6b_can_cast():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(can_cast(DType::INT32, DType::INT32, CastingMode::SAFE) == true)";
            std::cout << std::string("[FAIL] ") + description << std::endl;
            throw std::runtime_error(description);
        }
        if (!(can_cast(DType::FLOAT64, DType::FLOAT64, CastingMode::SAFE) == true)) {
            std::string description = std::string("np_test_phase6b_can_cast():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(can_cast(DType::FLOAT64, DType::FLOAT64, CastingMode::SAFE) == true)";
            std::cout << std::string("[FAIL] ") + description << std::endl;

is_open (np_test_3_all.cpp:11184)

      mmap1.setElementAt({ 1, 1 }, 42);
      mmap1.flush();

      // Move constructor
      auto mmap2 = std::move(mmap1);
      if (!(mmap2.getElementAt({ 1, 1 }) == 42)) {
          std::string description = std::string("np_test_io_memmap_move_semantics():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(mmap2.getElementAt({ 1, 1 }) == 42)";
          std::cout << std::string("[FAIL] ") + description << std::endl;
          throw std::runtime_error(description);
      }
      if (!(mmap2.is_open())) {
          std::string description = std::string("np_test_io_memmap_move_semantics():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(mmap2.is_open())";
          std::cout << std::string("[FAIL] ") + description << std::endl;
          throw std::runtime_error(description);
      }
      // std::cout << "[OK] Move constructor works" << std::endl;

      // Move assignment
      auto mmap3 = memmap::memmap<int32_t>(filename, { 3, 3 }, "r+");
      mmap3 = std::move(mmap2);
      if (!(mmap3.getElementAt({ 1, 1 }) == 42)) {

array (np_test_1_all.cpp:243)

    // std::cout << "Min: " << array.minArray() << std::endl;
    // std::cout << "Max: " << array.maxArray() << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

void testReshapeAndFlatten() {
    std::cout << "========= testReshapeAndFlatten =======================";

    auto range_array = NDArray<int32>::createRange(0, 12, 1);
    // std::cout << "Range array (0-11):" << std::endl;
    //range_array.printArray();

    auto reshaped = range_array.reshapeArray({3, 4});
    // std::cout << "Reshaped to 3x4:";
    //reshaped.printArray();

    auto flattened = reshaped.flattenArray();
    // std::cout << "Flattened:" << std::endl;
    //flattened.printArray();

array (np_test_1_all.cpp:243)

    // std::cout << "Min: " << array.minArray() << std::endl;
    // std::cout << "Max: " << array.maxArray() << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

void testReshapeAndFlatten() {
    std::cout << "========= testReshapeAndFlatten =======================";

    auto range_array = NDArray<int32>::createRange(0, 12, 1);
    // std::cout << "Range array (0-11):" << std::endl;
    //range_array.printArray();

    auto reshaped = range_array.reshapeArray({3, 4});
    // std::cout << "Reshaped to 3x4:";
    //reshaped.printArray();

    auto flattened = reshaped.flattenArray();
    // std::cout << "Flattened:" << std::endl;
    //flattened.printArray();

begin (np_test_1_all.cpp:6171)

    // Test sorted data generation
    std::vector<int> data = gen.generate(50, DataPattern::SORTED);

    if (!(data.size() == 50)) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(data.size() == 50)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Verify data is sorted
    if (!(std::is_sorted(data.begin(), data.end()))) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(std::is_sorted(data.begin(), data.end()))";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // std::cout << "Sorted data generation test passed" << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

begin (np_test_1_all.cpp:6171)

    // Test sorted data generation
    std::vector<int> data = gen.generate(50, DataPattern::SORTED);

    if (!(data.size() == 50)) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(data.size() == 50)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Verify data is sorted
    if (!(std::is_sorted(data.begin(), data.end()))) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(std::is_sorted(data.begin(), data.end()))";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // std::cout << "Sorted data generation test passed" << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

close (np_test_1_all.cpp:26157)

    void np_test_recarray_fromfile() {
      std::cout << "========= fromfile: create RecordArray from file =======================";

      // Create temporary test file
      std::string filename = "D:/Projects/Cpp2/temp/test_recarray.csv";
      std::ofstream outfile(filename);
      outfile << "1,10.5\n";
      outfile << "2,20.5\n";
      outfile << "3,30.5\n";
      outfile.close();

      // Define dtype
      std::vector<std::pair<std::string, numpy::DType>> fields = {
          {"id", numpy::DType::INT32},
          {"value", numpy::DType::FLOAT64}
      };
      auto dtype = std::make_shared<numpy::StructuredDType>(fields);

      // Load from file
      auto result = numpy::fromfile(filename, dtype, ",");

data (np_test_1_all.cpp:2084)

    }
    if (!(derived_ptr->extra == 2)) {
        std::string description = std::string("testObjectHolderEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(derived_ptr->extra == 2)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Test 6: Large object handling
    struct LargeObject {
        std::vector<double> data;
        LargeObject() : data(10000, 3.14159) {}  // Large object
    };

    LargeObject large;
    object_ large_obj(large);  // Should handle large objects
    if (!(!large_obj.is_null())) {
        std::string description = std::string("testObjectHolderEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(!large_obj.is_null())";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }
    if (!(large_obj.is_type<LargeObject>())) {

data (np_test_1_all.cpp:2084)

    }
    if (!(derived_ptr->extra == 2)) {
        std::string description = std::string("testObjectHolderEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(derived_ptr->extra == 2)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Test 6: Large object handling
    struct LargeObject {
        std::vector<double> data;
        LargeObject() : data(10000, 3.14159) {}  // Large object
    };

    LargeObject large;
    object_ large_obj(large);  // Should handle large objects
    if (!(!large_obj.is_null())) {
        std::string description = std::string("testObjectHolderEdgeCases():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(!large_obj.is_null())";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }
    if (!(large_obj.is_type<LargeObject>())) {

end (np_test_1_all.cpp:6171)

    // Test sorted data generation
    std::vector<int> data = gen.generate(50, DataPattern::SORTED);

    if (!(data.size() == 50)) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(data.size() == 50)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Verify data is sorted
    if (!(std::is_sorted(data.begin(), data.end()))) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(std::is_sorted(data.begin(), data.end()))";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // std::cout << "Sorted data generation test passed" << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

end (np_test_1_all.cpp:6171)

    // Test sorted data generation
    std::vector<int> data = gen.generate(50, DataPattern::SORTED);

    if (!(data.size() == 50)) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(data.size() == 50)";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // Verify data is sorted
    if (!(std::is_sorted(data.begin(), data.end()))) {
        std::string description = std::string("test_data_generator_sortedBenchmarkSorting():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(std::is_sorted(data.begin(), data.end()))";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }

    // std::cout << "Sorted data generation test passed" << std::endl;

    std::cout << " -> tests passed" << std::endl;
}

flush (np_test_2_all.cpp:10646)

      std::vector<size_t> shape = { 10, 5 };
      auto mmap_array = numpy::memmap::memmap<float>(TEMP_DIR + "test_memmap_new.npy", shape, "w+");

      // Write some data
      for (size_t i = 0; i < 10; ++i) {
        for (size_t j = 0; j < 5; ++j) {
          mmap_array.setElementAt({ i, j }, static_cast<float>(i * 5 + j + 1));
        }
      }

      mmap_array.flush();
      // std::cout << "[OK] Created and wrote to memory-mapped file\n";

      // Test reading from existing memory-mapped file
      auto mmap_readonly = numpy::memmap::memmap<float>(TEMP_DIR + "test_memmap_new.npy", shape, "r");

      for (size_t i = 0; i < 10; ++i) {
        for (size_t j = 0; j < 5; ++j) {
          float expected = static_cast<float>(i * 5 + j + 1);
          if (!(approx_equal(mmap_readonly.getElementAt({ i, j }), expected, 1e-6f))) {
              std::string description = std::string("testMemoryMapping():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(approx_equal(mmap_readonly.getElementAt({ i, j }), expected, 1e-6f))";

printArray (np_test_1_all.cpp:67)

        std::vector<size_t> indices(arr.getShape().size(), 0);
        do {
            if (arr.getElementAt(indices)) {
                return false;
            }
        } while (incrementIndices(indices, arr.getShape()));
        return true;
    }

    template<typename T>
    void printArray(const NDArray<T>& arr, const std::string& name) {
        std::cout << name << ": ";
        if (arr.getShape().size() == 1) {
            for (size_t i = 0; i < arr.getShape()[0]; ++i) {
                // std::cout << arr.getElementAt({i}) << " ";
            }
        }
        // std::cout << std::endl;
    }

    // Helper functions for bitwise operations tests

setElementAt (np_test_1_all.cpp:135)

    //ones_array.printArray();

    std::cout << " -> tests passed" << std::endl;
}

void testElementAccess() {
    std::cout << "========= testElementAccess =======================";

    auto array = createInt32Array({3, 3}, 0);

    array.setElementAt({0, 0}, 1);
    array.setElementAt({0, 1}, 2);
    array.setElementAt({0, 2}, 3);
    array.setElementAt({1, 1}, 5);
    array.setElementAt({2, 2}, 9);

    // std::cout << "Array after setting elements:" << std::endl;
    //array.printArray();

    // std::cout << "Element at (1,1): " << array.getElementAt({1, 1}) << std::endl;
    // std::cout << "Element at (2,2): " << array.getElementAt({2, 2});

size (np_test_1_all.cpp:47)

using namespace numpy;
using namespace numpy::benchmark;
using namespace numpy::char_;

// Helper functions for array comparison tests
namespace {
    const double TOLERANCE = 1e-10;

    bool allTrue(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);
        do {
            if (!arr.getElementAt(indices)) {
                return false;
            }
        } while (incrementIndices(indices, arr.getShape()));
        return true;
    }

    bool allFalse(const NDArray<bool>& arr) {
        std::vector<size_t> indices(arr.getShape().size(), 0);

toString (np_test_1_all.cpp:6826)

void testDatetime64CreationDateTime() {
    std::cout << "========= testDatetime64CreationDateTime =======================";

    // Test default constructor
    datetime64 default_dt;
    if (!(default_dt.isNaT())) {
        std::string description = std::string("testDatetime64CreationDateTime():") + __FILE__ + ":" + std::to_string(__LINE__) + ": !(default_dt.isNaT())";
        std::cout << std::string("[FAIL] ") + description << std::endl;
        throw std::runtime_error(description);
    }
    // std::cout << "Default datetime64 is NaT: " << default_dt.toString() << std::endl;

    // Test value constructor
    datetime64 epoch_day(0, DateTimeUnit::Day);
    // std::cout << "Epoch day: " << epoch_day.toString() << std::endl;

    // Test string constructor
    datetime64 date_from_string("2024-01-15");
    // std::cout << "Date from string '2024-01-15': " << date_from_string.toString() << std::endl;

    datetime64 datetime_from_string("2024-01-15T10:30:45");