Add Dynamic Memory Support to otLwf.sys (#1414)

* Add Dynamic Memory Support

* Use single list for free buffers. Null mNext on return of new buffer.

* Use Paged memory for Buffers.

examples/drivers/windows/include/openthread-core-windows-config.h

@@ -67,6 +67,16 @@
  */
 #define OPENTHREAD_CONFIG_MAX_JOINER_ENTRIES                16
 
+/**
+ * @def OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT
+ *
+ * The message pool is managed by platform defined logic when this flag is set.
+ * This feature would typically be used when operating in a multi-threaded system
+ * and multiple threads need to access the message pool.
+ *
+ */
+#define OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT       1
+
 /**
  * @def OPENTHREAD_CONFIG_LOG_LEVEL
  *

examples/drivers/windows/otLwf/eventprocessing.c

@@ -1090,28 +1090,7 @@ otLwfEventWorkerThread(
 
 exit:
 
-    if (pFilter->otCtx != NULL)
-    {
-        otInstanceFinalize(pFilter->otCtx);
-        pFilter->otCtx = NULL;
-        
-#if DEBUG_ALLOC
-        {
-            NT_ASSERT(pFilter->otOutstandingAllocationCount == 0);
-            NT_ASSERT(pFilter->otOutstandingMemoryAllocated == 0);
-            PLIST_ENTRY Link = pFilter->otOutStandingAllocations.Flink;
-            while (Link != &pFilter->otOutStandingAllocations)
-            {
-                OT_ALLOC* AllocHeader = CONTAINING_RECORD(Link, OT_ALLOC, Link);
-                Link = Link->Flink;
-
-                LogVerbose(DRIVER_DEFAULT, "Leaked Alloc ID:%u", AllocHeader->ID);
-            
-                ExFreePoolWithTag(AllocHeader, 'OTDM');
-            }
-        }
-#endif
-    }
+    otLwfReleaseInstance(pFilter);
 
     if (pFilter->otInstanceBuffer != NULL)
     {

examples/drivers/windows/otLwf/filter.h

@@ -111,6 +111,29 @@ typedef enum OTLWF_DEVICE_STATUS
 #define OT_EVENT_TIMER_RUNNING      1
 #define OT_EVENT_TIMER_FIRED        2
 
+#if OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT
+
+typedef struct BufferPool
+{
+    struct BufferPool* Next;
+    uint8_t Buffers[0];
+
+} BufferPool;
+
+enum
+{
+    kPageSize                = PAGE_SIZE,
+    kPagesPerBufferPool      = 1,
+    kMaxPagesForBufferPools  = 64,
+    kMaxBytesForBufferPools  = kPageSize * kMaxPagesForBufferPools,
+
+    kEstimatedBufferSize     = 128,         // sizeof(Thread::Buffer)
+    kEstimatedBufferPoolSize = ((kPageSize * kPagesPerBufferPool) - sizeof(BufferPool)) / kEstimatedBufferSize,
+    kEstimatedMaxBuffers     = kMaxPagesForBufferPools * kEstimatedBufferPoolSize
+};
+
+#endif
+
 //
 // Define the filter struct
 //
@@ -244,6 +267,15 @@ typedef struct _MS_FILTER
 
         BOOLEAN                     otPendingMacOffloadEnabled;
 
+#if OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT
+        uint16_t                    otBufferSize;               // Bytes in a single buffer
+        uint16_t                    otBufferPoolByteSize;       // Bytes in a buffer pool
+        uint16_t                    otBufferPoolBufferCount;    // Number of buffers in a pool
+        uint16_t                    otBuffersLeft;              // Number of buffers left to return
+        BufferPool*                 otBufferPoolHead;           // List of buffer pools
+        otMessage*                  otFreeBuffers;              // List of buffers to return
+#endif
+
 #if DEBUG_ALLOC
         // Used for tracking memory allocations
         HANDLE                      otThreadId;

examples/drivers/windows/otLwf/precomp.h

@@ -76,6 +76,7 @@ RtlCopyBufferToMdl(
 #include <openthread/platform/misc.h>
 #include <openthread/platform/alarm.h>
 #include <openthread/platform/settings.h>
+#include <openthread/platform/messagepool.h>
 #include <ncp/spinel.h>
 
 #include <otLwfIoctl.h>

examples/drivers/windows/otLwf/radio.c

@@ -65,8 +65,7 @@ otPlatReset(
     (void)otLwfCmdResetDevice(pFilter, TRUE);
 
     // Finalize previous OpenThread instance
-    otInstanceFinalize(pFilter->otCtx);
-    pFilter->otCtx = NULL;
+    otLwfReleaseInstance(pFilter);
 
     // Reset radio layer
     pFilter->otPhyState = kStateDisabled;

examples/drivers/windows/otLwf/thread.c

@@ -208,17 +208,69 @@ otLwfFindFromCurrentThread()
 }
 #endif
 
+#define OTPLAT_CALLOC_TAG 'OTDM'
+#define BUFFER_POOL_TAG 'OTBP'
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+otLwfReleaseInstance(
+    _In_ PMS_FILTER pFilter
+    )
+{
+    LogFuncEntry(DRIVER_DEFAULT);
+
+    if (pFilter->otCtx != NULL)
+    {
+        otInstanceFinalize(pFilter->otCtx);
+        pFilter->otCtx = NULL;
+
+#if OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT
+
+        // Free all the pools as there should be no outstanding
+        // references to the buffers any more.
+        BufferPool *curPool = pFilter->otBufferPoolHead;
+        while (curPool != NULL)
+        {
+            BufferPool *nextPool = curPool->Next;
+            ExFreePoolWithTag(curPool, BUFFER_POOL_TAG);
+            curPool = nextPool;
+        }
+
+#endif
+
+#if DEBUG_ALLOC
+
+        NT_ASSERT(pFilter->otOutstandingAllocationCount == 0);
+        NT_ASSERT(pFilter->otOutstandingMemoryAllocated == 0);
+        PLIST_ENTRY Link = pFilter->otOutStandingAllocations.Flink;
+        while (Link != &pFilter->otOutStandingAllocations)
+        {
+            OT_ALLOC* AllocHeader = CONTAINING_RECORD(Link, OT_ALLOC, Link);
+            Link = Link->Flink;
+
+            LogVerbose(DRIVER_DEFAULT, "Leaked Alloc ID:%u", AllocHeader->ID);
+
+            ExFreePoolWithTag(AllocHeader, OTPLAT_CALLOC_TAG);
+        }
+
+#endif
+    }
+
+    LogFuncExit(DRIVER_DEFAULT);
+}
+
 //
 // OpenThread Platform functions
 //
 
+_IRQL_requires_max_(PASSIVE_LEVEL)
 void *otPlatCAlloc(size_t aNum, size_t aSize)
 {
     size_t totalSize = aNum * aSize;
 #if DEBUG_ALLOC
     totalSize += sizeof(OT_ALLOC);
 #endif
-    PVOID mem = ExAllocatePoolWithTag(NonPagedPoolNx, totalSize, 'OTDM');
+    PVOID mem = ExAllocatePoolWithTag(PagedPool, totalSize, OTPLAT_CALLOC_TAG);
     if (mem)
     {
         RtlZeroMemory(mem, totalSize);
@@ -240,7 +292,8 @@ void *otPlatCAlloc(size_t aNum, size_t aSize)
     return mem;
 }
 
-void otPlatFree(void *aPtr)
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void otPlatFree(_In_opt_ void *aPtr)
 {
     if (aPtr == NULL) return;
 #if DEBUG_ALLOC
@@ -253,9 +306,124 @@ void otPlatFree(void *aPtr)
     InterlockedAdd(&pFilter->otOutstandingMemoryAllocated, -AllocHeader->Length);
     RemoveEntryList(&AllocHeader->Link);
 #endif
-    ExFreePoolWithTag(aPtr, 'OTDM');
+    ExFreePoolWithTag(aPtr, OTPLAT_CALLOC_TAG);
 }
 
+#if OPENTHREAD_CONFIG_PLATFORM_MESSAGE_MANAGEMENT
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BufferPool* AllocBufferPool(_In_ PMS_FILTER pFilter)
+{
+    // Allocate the memory
+    BufferPool* bufPool = (BufferPool*)ExAllocatePoolWithTag(PagedPool, pFilter->otBufferPoolByteSize, BUFFER_POOL_TAG);
+    if (bufPool == NULL)
+    {
+        LogWarning(DRIVER_DEFAULT, "Failed to allocate new buffer pool!");
+        return NULL;
+    }
+
+    // Zero out the memory
+    RtlZeroMemory(bufPool, pFilter->otBufferPoolByteSize);
+
+    // Set all mNext for the buffers
+    otMessage* prevBuf = (otMessage*)bufPool->Buffers;
+    for (uint16_t i = 1; i < pFilter->otBufferPoolBufferCount; i++)
+    {
+        otMessage* curBuf =
+            (otMessage*)&bufPool->Buffers[i * pFilter->otBufferSize];
+
+        prevBuf->mNext = curBuf;
+        prevBuf = curBuf;
+    }
+
+    LogVerbose(DRIVER_DEFAULT, "Allocated new buffer pool (%d bytes)!", pFilter->otBufferPoolByteSize);
+
+    return bufPool;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+otMessage* GetNextFreeBufferFromPool(_In_ PMS_FILTER pFilter)
+{
+    // Immediately return if we have hit our limit
+    if (pFilter->otBuffersLeft == 0) return NULL;
+
+    // If we don't have any free buffers left, allocate another pool
+    if (pFilter->otFreeBuffers == NULL)
+    {
+        BufferPool *newPool = AllocBufferPool(pFilter);
+        if (newPool == NULL) return NULL; // Out of physical memory
+
+        // Push on top of the pool list
+        newPool->Next = pFilter->otBufferPoolHead;
+        pFilter->otBufferPoolHead = newPool;
+
+        // Set the free buffer list
+        pFilter->otFreeBuffers = (otMessage*)newPool->Buffers;
+    }
+
+    // Pop the top free buffer
+    otMessage* buffer = pFilter->otFreeBuffers;
+    pFilter->otFreeBuffers = pFilter->otFreeBuffers->mNext;
+    pFilter->otBuffersLeft--;
+    buffer->mNext = NULL;
+    return buffer;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void otPlatMessagePoolInit(_In_ otInstance *otCtx, uint16_t aMinNumFreeBuffers, size_t aBufferSize)
+{
+    NT_ASSERT(otCtx);
+    PMS_FILTER pFilter = otCtxToFilter(otCtx);
+
+    LogFuncEntry(DRIVER_DEFAULT);
+    UNREFERENCED_PARAMETER(aMinNumFreeBuffers);
+
+    // Initialize parameters
+    pFilter->otBufferSize = (uint16_t)aBufferSize;
+    pFilter->otBufferPoolByteSize = (uint16_t)(kPageSize * kPagesPerBufferPool);
+    pFilter->otBufferPoolBufferCount = (uint16_t)((pFilter->otBufferPoolByteSize - sizeof(BufferPool)) / aBufferSize);
+    pFilter->otBuffersLeft = kMaxPagesForBufferPools * pFilter->otBufferPoolBufferCount;
+
+    // Allocate first pool
+    pFilter->otBufferPoolHead = AllocBufferPool(pFilter);
+    ASSERT(pFilter->otBufferPoolHead); // Should this API allow for failure ???
+
+    // Set initial free buffer list
+    pFilter->otFreeBuffers = (otMessage*)pFilter->otBufferPoolHead->Buffers;
+
+    LogFuncExit(DRIVER_DEFAULT);
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+otMessage *otPlatMessagePoolNew(_In_ otInstance *otCtx)
+{
+    NT_ASSERT(otCtx);
+    PMS_FILTER pFilter = otCtxToFilter(otCtx);
+    return GetNextFreeBufferFromPool(pFilter);
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void otPlatMessagePoolFree(_In_ otInstance *otCtx, _In_ otMessage *aBuffer)
+{
+    NT_ASSERT(otCtx);
+    PMS_FILTER pFilter = otCtxToFilter(otCtx);
+
+    // Put buffer back on the list
+    aBuffer->mNext = pFilter->otFreeBuffers;
+    pFilter->otFreeBuffers = aBuffer;
+    pFilter->otBuffersLeft++;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+uint16_t otPlatMessagePoolNumFreeBuffers(_In_ otInstance *otCtx)
+{
+    NT_ASSERT(otCtx);
+    PMS_FILTER pFilter = otCtxToFilter(otCtx);
+    return pFilter->otBuffersLeft;
+}
+
+#endif
+
 uint32_t otPlatRandomGet()
 {
     LARGE_INTEGER Counter = KeQueryPerformanceCounter(NULL);

examples/drivers/windows/otLwf/thread.h

@@ -63,6 +63,16 @@ otLwfUninitializeThreadMode(
     _In_ PMS_FILTER pFilter
     );
 
+//
+// Clean up otInstance
+//
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+otLwfReleaseInstance(
+    _In_ PMS_FILTER pFilter
+    );
+
 //
 // Event Processing Functions
 //