pagepae.c

Go to the documentation of this file.

/*
 * COPYRIGHT:       See COPYING in the top level directory
 * PROJECT:         ReactOS kernel
 * FILE:            ntoskrnl/mm/i386/page.c
 * PURPOSE:         Low level memory managment manipulation
 *
 * PROGRAMMERS:     David Welch (welch@cwcom.net)
 */

/* INCLUDES ***************************************************************/

#include <ntoskrnl.h>
#define NDEBUG
#include <debug.h>

#if defined (ALLOC_PRAGMA)
#pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
#pragma alloc_text(INIT, MiInitPageDirectoryMap)
#endif


/* GLOBALS *****************************************************************/

#define PA_BIT_PRESENT   (0)
#define PA_BIT_READWRITE (1)
#define PA_BIT_USER      (2)
#define PA_BIT_WT        (3)
#define PA_BIT_CD        (4)
#define PA_BIT_ACCESSED  (5)
#define PA_BIT_DIRTY     (6)
#define PA_BIT_GLOBAL    (8)

#define PA_PRESENT   (1 << PA_BIT_PRESENT)
#define PA_READWRITE (1 << PA_BIT_READWRITE)
#define PA_USER      (1 << PA_BIT_USER)
#define PA_DIRTY     (1 << PA_BIT_DIRTY)
#define PA_WT        (1 << PA_BIT_WT)
#define PA_CD        (1 << PA_BIT_CD)
#define PA_ACCESSED  (1 << PA_BIT_ACCESSED)
#define PA_GLOBAL    (1 << PA_BIT_GLOBAL)

#define PAGETABLE_MAP           (0xc0000000)
#define PAGEDIRECTORY_MAP       (0xc0000000 + (PAGETABLE_MAP / (1024)))

#define PAE_PAGEDIRECTORY_MAP   (0xc0000000 + (PAGETABLE_MAP / (512)))

#define HYPERSPACE              (Ke386Pae ? 0xc0800000 : 0xc0400000)
#define IS_HYPERSPACE(v)        (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))

ULONG MmGlobalKernelPageDirectory[1024];
ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];

#define PTE_TO_PFN(X)  ((X) >> PAGE_SHIFT)
#define PFN_TO_PTE(X)  ((X) << PAGE_SHIFT)

#define PAE_PTE_TO_PFN(X)   (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
#define PAE_PFN_TO_PTE(X)   ((X) << PAGE_SHIFT)

#if defined(__GNUC__)
#define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
#else
__inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
{
   LARGE_INTEGER dummy;
   dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
   return dummy;
}
#endif

extern BOOLEAN Ke386Pae;
extern BOOLEAN Ke386NoExecute;

/* FUNCTIONS ***************************************************************/

BOOLEAN MmUnmapPageTable(PULONG Pt);

ULONG_PTR
NTAPI
MiFlushTlbIpiRoutine(ULONG_PTR Address)
{
   if (Address == (ULONGLONG)-1)
   {
      KeFlushCurrentTb();
   }
   else if (Address == (ULONGLONG)-2)
   {
      KeFlushCurrentTb();
   }
   else
   {
       __invlpg((PVOID)Address);
   }
   return 0;
}

VOID
MiFlushTlb(PULONG Pt, PVOID Address)
{
#ifdef CONFIG_SMP
   if (Pt)
   {
      MmUnmapPageTable(Pt);
   }
   if (KeNumberProcessors > 1)
   {
      KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address);
   }
   else
   {
      MiFlushTlbIpiRoutine((ULONG_PTR)Address);
   }
#else
   if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
   {
      __invlpg(Address);
   }
#endif
}



PULONG
MmGetPageDirectory(VOID)
{
   return (PULONG)__readcr3();
}

static ULONG
ProtectToPTE(ULONG flProtect)
{
   ULONG Attributes = 0;

   if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
   {
      Attributes = 0;
   }
   else if (flProtect & PAGE_IS_WRITABLE)
   {
      Attributes = PA_PRESENT | PA_READWRITE;
   }
   else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
   {
      Attributes = PA_PRESENT;
   }
   else
   {
      DPRINT1("Unknown main protection type.\n");
      ASSERT(FALSE);
   }
   if (Ke386NoExecute &&
       !(flProtect & PAGE_IS_EXECUTABLE))
   {
      Attributes = Attributes | 0x80000000;
   }

   if (flProtect & PAGE_SYSTEM)
   {
   }
   else
   {
      Attributes = Attributes | PA_USER;
   }
   if (flProtect & PAGE_NOCACHE)
   {
      Attributes = Attributes | PA_CD;
   }
   if (flProtect & PAGE_WRITETHROUGH)
   {
      Attributes = Attributes | PA_WT;
   }
   return(Attributes);
}

#define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))

#define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
                                ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
#define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))

#define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))

#define ADDR_TO_PTE_OFFSET(v)  ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)


#define PAE_ADDR_TO_PAGE_TABLE(v)   (((ULONG)(v)) / (512 * PAGE_SIZE))

#define PAE_ADDR_TO_PDE(v)          (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
                                                  ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
#define PAE_ADDR_TO_PTE(v)          (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))


#define PAE_ADDR_TO_PDTE_OFFSET(v)  (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))

#define PAE_ADDR_TO_PDE_PAGE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))

#define PAE_ADDR_TO_PDE_OFFSET(v)   (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))

#define PAE_ADDR_TO_PTE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)


NTSTATUS
NTAPI
Mmi386ReleaseMmInfo(PEPROCESS Process)
{
   PUSHORT LdtDescriptor;
   ULONG LdtBase;
   ULONG i, j;

   DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);

   LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor;
   LdtBase = LdtDescriptor[1] |
             ((LdtDescriptor[2] & 0xff) << 16) |
             ((LdtDescriptor[3] & ~0xff) << 16);

   DPRINT("LdtBase: %x\n", LdtBase);

   if (LdtBase)
   {
      ExFreePool((PVOID) LdtBase);
   }

   if (Ke386Pae)
   {
      PULONGLONG PageDirTable;
      PULONGLONG PageDir;
      PULONGLONG Pde;
      ULONG k;

      PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
      for (i = 0; i < 4; i++)
      {
         PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
         if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart))
         {
            for (j = 0; j < 512; j++)
            {
               if (PageDir[j] != 0LL)
               {
                  DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
                          Process->UniqueProcessId,
                          (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
                          ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]);
                  Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
                  for (k = 0; k < 512; k++)
                  {
                     if(Pde[k] != 0)
                     {
                        if (Pde[k] & PA_PRESENT)
                        {
                           DPRINT1("Page at %08x is not freed\n",
                                   (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
                        }
                        else
                        {
                           DPRINT1("Swapentry %x at %x is not freed\n",
                                   (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
                        }
                     }
                  }
                  MmDeleteHyperspaceMapping(Pde);
                  MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
               }
            }
         }
         if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
         {
            MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
            MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
         }
         MmDeleteHyperspaceMapping(PageDir);
         MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
      }
      MmDeleteHyperspaceMapping((PVOID)PageDirTable);
      MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
   }
   else
   {
      PULONG Pde;
      PULONG PageDir;
      PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
      for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++)
      {
         if (PageDir[i] != 0)
         {
            DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
                    i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
                    ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]);
            Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
            for (j = 0; j < 1024; j++)
            {
               if(Pde[j] != 0)
               {
                  if (Pde[j] & PA_PRESENT)
                  {
                     DPRINT1("Page at %08x is not freed\n",
                             i * 4 * 1024 * 1024 + j * PAGE_SIZE);
                  }
                  else
                  {
                     DPRINT1("Swapentry %x at %x is not freed\n",
                             Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
                  }
               }
            }
            MmDeleteHyperspaceMapping(Pde);
            MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
         }
      }
      MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
      MmDeleteHyperspaceMapping(PageDir);
      MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
   }

#if defined(__GNUC__)

   Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
#else

   Process->Pcb.DirectoryTableBase.QuadPart = 0;
#endif

   DPRINT("Finished Mmi386ReleaseMmInfo()\n");
   return(STATUS_SUCCESS);
}

NTSTATUS
NTAPI
MmInitializeHandBuiltProcess(IN PEPROCESS Process,
                             IN PLARGE_INTEGER DirectoryTableBase)
{
    /* Share the directory base with the idle process */
    *DirectoryTableBase = PsGetCurrentProcess()->Pcb.DirectoryTableBase;

    /* Initialize the Addresss Space */
    MmInitializeAddressSpace(Process, (PMADDRESS_SPACE)&Process->VadRoot);

    /* The process now has an address space */
    Process->HasAddressSpace = TRUE;
    return STATUS_SUCCESS;
}

BOOLEAN
NTAPI
MmCreateProcessAddressSpace(IN ULONG MinWs,
                            IN PEPROCESS Process,
                            IN PLARGE_INTEGER DirectoryTableBase)
{
   NTSTATUS Status;
   ULONG i, j;
   PFN_NUMBER Pfn[7];
   ULONG Count;

   DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process);

   Count = Ke386Pae ? 7 : 2;

   for (i = 0; i < Count; i++)
   {
      Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
      if (!NT_SUCCESS(Status))
      {
          for (j = 0; j < i; j++)
          {
              MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
          }

          return FALSE;
      }
   }

   if (Ke386Pae)
   {
      PULONGLONG PageDirTable;
      PULONGLONG PageDir;

      PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
      for (i = 0; i < 4; i++)
      {
         PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
      }
      MmDeleteHyperspaceMapping(PageDirTable);
      for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
      {
         PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
         memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
         if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
         {
            for (j = 0; j < 4; j++)
            {
               PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
            }
         }
         if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
         {
            PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
            PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
         }
         MmDeleteHyperspaceMapping(PageDir);
      }
   }
   else
   {
      PULONG PageDirectory;
      PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);

      memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
             MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
             (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));

      DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
      PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
      PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;

      MmDeleteHyperspaceMapping(PageDirectory);
   }

   DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
   DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
   return TRUE;
}

VOID
NTAPI
MmDeletePageTable(PEPROCESS Process, PVOID Address)
{
   PEPROCESS CurrentProcess = PsGetCurrentProcess();

   if (Process != NULL && Process != CurrentProcess)
   {
      KeAttachProcess(&Process->Pcb);
   }

   if (Ke386Pae)
   {
      ULONGLONG ZeroPde = 0LL;
      (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
      MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
   }
   else
   {
      *(ADDR_TO_PDE(Address)) = 0;
      MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
   }
   if (Address >= MmSystemRangeStart)
   {
      ASSERT(FALSE);
      //       MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
   }
   if (Process != NULL && Process != CurrentProcess)
   {
      KeDetachProcess();
   }
}

VOID
NTAPI
MmFreePageTable(PEPROCESS Process, PVOID Address)
{
   PEPROCESS CurrentProcess = PsGetCurrentProcess();
   ULONG i;
   PFN_NUMBER Pfn;

   DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
   if (Process != NULL && Process != CurrentProcess)
   {
      KeAttachProcess(&Process->Pcb);
   }
   if (Ke386Pae)
   {
      PULONGLONG PageTable;
      ULONGLONG ZeroPte = 0LL;
      PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
      for (i = 0; i < 512; i++)
      {
         if (PageTable[i] != 0LL)
         {
            DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
                     ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
            ASSERT(FALSE);
         }
      }
      Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
      (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
      MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
   }
   else
   {
      PULONG PageTable;
      PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
      for (i = 0; i < 1024; i++)
      {
         if (PageTable[i] != 0)
         {
            DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
                     ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
            ASSERT(FALSE);
         }
      }
      Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
      *(ADDR_TO_PDE(Address)) = 0;
      MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
   }

   if (Address >= MmSystemRangeStart)
   {
      //    MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
      ASSERT(FALSE);
   }
   else
   {
      MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
   }
   if (Process != NULL && Process != CurrentProcess)
   {
      KeDetachProcess();
   }
}

static PULONGLONG
MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
{
   NTSTATUS Status;
   PFN_NUMBER Pfn;
   ULONGLONG Entry;
   ULONGLONG ZeroEntry = 0LL;
   PULONGLONG Pt;
   PULONGLONG PageDir;
   PULONGLONG PageDirTable;

   DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
          Process, Address, Create);
   if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
   {
      ASSERT(FALSE);
   }
   if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
   {
      PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
      if (PageDirTable == NULL)
      {
         ASSERT(FALSE);
      }
      PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
      MmDeleteHyperspaceMapping(PageDirTable);
      if (PageDir == NULL)
      {
         ASSERT(FALSE);
      }
      PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
      Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
      if (Entry == 0LL)
      {
         if (Create == FALSE)
         {
            MmDeleteHyperspaceMapping(PageDir);
            return NULL;
         }
         Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
         if (!NT_SUCCESS(Status))
         {
            ASSERT(FALSE);
         }
         Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
         Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
         if (Entry != 0LL)
         {
            MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
            Pfn = PAE_PTE_TO_PFN(Entry);
         }
      }
      else
      {
         Pfn = PAE_PTE_TO_PFN(Entry);
      }
      MmDeleteHyperspaceMapping(PageDir);
      Pt = MmCreateHyperspaceMapping(Pfn);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
   }
   PageDir = PAE_ADDR_TO_PDE(Address);
   if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
   {
      if (Address >= MmSystemRangeStart)
      {
         if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
         {
            if (Create == FALSE)
            {
               return NULL;
            }
            Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
            if (!NT_SUCCESS(Status))
            {
               ASSERT(FALSE);
            }
            Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
            if (Ke386GlobalPagesEnabled)
            {
               Entry |= PA_GLOBAL;
            }
            if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
            {
               MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
            }
         }
         (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
      }
      else
      {
         if (Create == FALSE)
         {
            return NULL;
         }
         Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
         if (!NT_SUCCESS(Status))
         {
            ASSERT(FALSE);
         }
         Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
         Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
         if (Entry != 0LL)
         {
            MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
         }
      }
   }
   return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
}

static PULONG
MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
{
   ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
   NTSTATUS Status;
   PFN_NUMBER Pfn;
   ULONG Entry;
   PULONG Pt, PageDir;

   if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
   {
      PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart));
      if (PageDir == NULL)
      {
         ASSERT(FALSE);
      }
      if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
      {
         if (Create == FALSE)
         {
            MmDeleteHyperspaceMapping(PageDir);
            return NULL;
         }
         Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
         if (!NT_SUCCESS(Status) || Pfn == 0)
         {
            ASSERT(FALSE);
         }
         Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
         if (Entry != 0)
         {
            MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
            Pfn = PTE_TO_PFN(Entry);
         }
      }
      else
      {
         Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
      }
      MmDeleteHyperspaceMapping(PageDir);
      Pt = MmCreateHyperspaceMapping(Pfn);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      return Pt + ADDR_TO_PTE_OFFSET(Address);
   }
   PageDir = ADDR_TO_PDE(Address);
   if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
   {
      if (Address >= MmSystemRangeStart)
      {
         if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
         {
            if (Create == FALSE)
            {
               return NULL;
            }
            Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
            if (!NT_SUCCESS(Status) || Pfn == 0)
            {
               ASSERT(FALSE);
            }
            Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
            if (Ke386GlobalPagesEnabled)
            {
               Entry |= PA_GLOBAL;
            }
            if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
            {
               MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
            }
         }
         (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
      }
      else
      {
         if (Create == FALSE)
         {
            return NULL;
         }
         Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
         if (!NT_SUCCESS(Status) || Pfn == 0)
         {
            ASSERT(FALSE);
         }
         Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
         if (Entry != 0)
         {
            MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
         }
      }
   }
   return (PULONG)ADDR_TO_PTE(Address);
}

BOOLEAN MmUnmapPageTable(PULONG Pt)
{
   if (Ke386Pae)
   {
      if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
      {
         return TRUE;
      }
   }
   else
   {
      if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
      {
         return TRUE;
      }
   }
   if (Pt)
   {
      MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
   }
   return FALSE;
}

static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
{
   ULONGLONG Pte;
   PULONGLONG Pt;

   Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
   if (Pt)
   {
      Pte = *Pt;
      MmUnmapPageTable((PULONG)Pt);
      return Pte;
   }
   return 0;
}

static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
{
   ULONG Pte;
   PULONG Pt;

   Pt = MmGetPageTableForProcess(Process, Address, FALSE);
   if (Pt)
   {
      Pte = *Pt;
      MmUnmapPageTable(Pt);
      return Pte;
   }
   return 0;
}

PFN_NUMBER
NTAPI
MmGetPfnForProcess(PEPROCESS Process,
                   PVOID Address)
{

   if (Ke386Pae)
   {
      ULONGLONG Entry;
      Entry = MmGetPageEntryForProcessForPAE(Process, Address);
      if (!(Entry & PA_PRESENT))
      {
         return 0;
      }
      return(PAE_PTE_TO_PFN(Entry));
   }
   else
   {
      ULONG Entry;
      Entry = MmGetPageEntryForProcess(Process, Address);
      if (!(Entry & PA_PRESENT))
      {
         return 0;
      }
      return(PTE_TO_PFN(Entry));
   }
}

VOID
NTAPI
MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_NUMBER Page)
/*
 * FUNCTION: Delete a virtual mapping
 */
{
   BOOLEAN WasValid;
   if (Ke386Pae)
   {
      ULONGLONG Pte;
      ULONGLONG tmpPte;
      PULONGLONG Pt;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      /*
       * Atomically disable the present bit and get the old value.
       */
      do
      {
         Pte = *Pt;
         tmpPte = Pte & ~PA_PRESENT;
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));

      MiFlushTlb((PULONG)Pt, Address);
      WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
      if (!WasValid)
      {
         ASSERT(FALSE);
      }

      /*
       * Return some information to the caller
       */
      if (WasDirty != NULL)
      {
         *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
      }
      if (Page != NULL)
      {
         *Page = PAE_PTE_TO_PFN(Pte);
      }
   }
   else
   {
      ULONG Pte;
      PULONG Pt;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      /*
       * Atomically disable the present bit and get the old value.
       */
      do
      {
        Pte = *Pt;
      } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));

      MiFlushTlb(Pt, Address);
      WasValid = (PAGE_MASK(Pte) != 0);
      if (!WasValid)
      {
         ASSERT(FALSE);
      }

      /*
       * Return some information to the caller
       */
      if (WasDirty != NULL)
      {
         *WasDirty = Pte & PA_DIRTY;
      }
      if (Page != NULL)
      {
         *Page = PTE_TO_PFN(Pte);
      }
   }
}

VOID
NTAPI
MmRawDeleteVirtualMapping(PVOID Address)
{
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG ZeroPte = 0LL;
      Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
      if (Pt)
      {
         /*
          * Set the entry to zero
          */
         (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte);
         MiFlushTlb((PULONG)Pt, Address);
      }
   }
   else
   {
      PULONG Pt;

      Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
      if (Pt && *Pt)
      {
         /*
          * Set the entry to zero
          */
         (void)InterlockedExchangeUL(Pt, 0);
         MiFlushTlb(Pt, Address);
      }
   }
}

VOID
NTAPI
MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage,
                       BOOLEAN* WasDirty, PPFN_NUMBER Page)
/*
 * FUNCTION: Delete a virtual mapping
 */
{
   BOOLEAN WasValid = FALSE;
   PFN_NUMBER Pfn;

   DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
          Process, Address, FreePage, WasDirty, Page);
   if (Ke386Pae)
   {
      ULONGLONG Pte;
      PULONGLONG Pt;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         if (WasDirty != NULL)
         {
            *WasDirty = FALSE;
         }
         if (Page != NULL)
         {
            *Page = 0;
         }
         return;
      }

      /*
       * Atomically set the entry to zero and get the old value.
       */
      Pte = 0LL;
      Pte = ExfpInterlockedExchange64UL(Pt, &Pte);

      MiFlushTlb((PULONG)Pt, Address);

      WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
      if (WasValid)
      {
         Pfn = PAE_PTE_TO_PFN(Pte);
         MmMarkPageUnmapped(Pfn);
      }
      else
      {
         Pfn = 0;
      }

      if (FreePage && WasValid)
      {
         MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
      }

      /*
       * Return some information to the caller
       */
      if (WasDirty != NULL)
      {
         *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
      }
      if (Page != NULL)
      {
         *Page = Pfn;
      }
   }
   else
   {
      ULONG Pte;
      PULONG Pt;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);

      if (Pt == NULL)
      {
         if (WasDirty != NULL)
         {
            *WasDirty = FALSE;
         }
         if (Page != NULL)
         {
            *Page = 0;
         }
         return;
      }

      /*
       * Atomically set the entry to zero and get the old value.
       */
      Pte = InterlockedExchangeUL(Pt, 0);

      MiFlushTlb(Pt, Address);

      WasValid = (PAGE_MASK(Pte) != 0);
      if (WasValid)
      {
         Pfn = PTE_TO_PFN(Pte);
         MmMarkPageUnmapped(Pfn);
      }
      else
      {
         Pfn = 0;
      }

      if (FreePage && WasValid)
      {
         MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
      }

      /*
       * Return some information to the caller
       */
      if (WasDirty != NULL)
      {
         *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
      }
      if (Page != NULL)
      {
         *Page = Pfn;
      }
   }
   /*
    * Decrement the reference count for this page table.
    */
   if (Process != NULL && WasValid &&
       ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
       Address < MmSystemRangeStart)
   {
      PUSHORT Ptrc;
      ULONG Idx;

      Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
      Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);

      Ptrc[Idx]--;
      if (Ptrc[Idx] == 0)
      {
         MmFreePageTable(Process, Address);
      }
   }
}

VOID
NTAPI
MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
                        SWAPENTRY* SwapEntry)
/*
 * FUNCTION: Delete a virtual mapping
 */
{
   if (Ke386Pae)
   {
      ULONGLONG Pte;
      PULONGLONG Pt;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         *SwapEntry = 0;
         return;
      }

      /*
       * Atomically set the entry to zero and get the old value.
       */
      Pte = 0LL;
      Pte = ExfpInterlockedExchange64UL(Pt, &Pte);

      MiFlushTlb((PULONG)Pt, Address);

      /*
       * Decrement the reference count for this page table.
       */
      if (Process != NULL && Pte &&
          ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
          Address < MmSystemRangeStart)
      {
         PUSHORT Ptrc;

         Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;

         Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
         if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
         {
            MmFreePageTable(Process, Address);
         }
      }


      /*
       * Return some information to the caller
       */
      *SwapEntry = Pte >> 1;
   }
   else
   {
      ULONG Pte;
      PULONG Pt;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);

      if (Pt == NULL)
      {
         *SwapEntry = 0;
         return;
      }

      /*
       * Atomically set the entry to zero and get the old value.
       */
      Pte = InterlockedExchangeUL(Pt, 0);

      MiFlushTlb(Pt, Address);

      /*
       * Decrement the reference count for this page table.
       */
      if (Process != NULL && Pte &&
          ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
          Address < MmSystemRangeStart)
      {
         PUSHORT Ptrc;

         Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;

         Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
         if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
         {
            MmFreePageTable(Process, Address);
         }
      }


      /*
       * Return some information to the caller
       */
      *SwapEntry = Pte >> 1;
   }
}

BOOLEAN
Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
{
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      PULONGLONG Pde;
      Pde = PAE_ADDR_TO_PDE(PAddress);
      if (*Pde == 0LL)
      {
         Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
#if 0
         /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
         FLASH_TLB_ONE(PAddress);
#endif
         if (Pt != NULL)
         {
            return TRUE;
         }
      }
   }
   else
   {
      PULONG Pt, Pde;
      Pde = ADDR_TO_PDE(PAddress);
      if (*Pde == 0)
      {
         Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
#if 0
         /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
         FLASH_TLB_ONE(PAddress);
#endif
         if (Pt != NULL)
         {
            return TRUE;
         }
      }
   }
   return(FALSE);
}

BOOLEAN
NTAPI
MmIsDirtyPage(PEPROCESS Process, PVOID Address)
{
   if (Ke386Pae)
   {
      return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
   }
   else
   {
      return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
   }
}

BOOLEAN
NTAPI
MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
{
   if (Address < MmSystemRangeStart && Process == NULL)
   {
      DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
      ASSERT(FALSE);
   }
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG Pte;
      ULONGLONG tmpPte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
         tmpPte = Pte & ~PA_ACCESSED;
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));

      if (Pte & PA_ACCESSED)
      {
         MiFlushTlb((PULONG)Pt, Address);
         return TRUE;
      }
      else
      {
         MmUnmapPageTable((PULONG)Pt);
         return FALSE;
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
      } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));

      if (Pte & PA_ACCESSED)
      {
         MiFlushTlb(Pt, Address);
         return TRUE;
      }
      else
      {
         MmUnmapPageTable(Pt);
         return FALSE;
      }
   }
}

VOID
NTAPI
MmSetCleanPage(PEPROCESS Process, PVOID Address)
{
   if (Address < MmSystemRangeStart && Process == NULL)
   {
      DPRINT1("MmSetCleanPage is called for user space without a process.\n");
      ASSERT(FALSE);
   }
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG Pte;
      ULONGLONG tmpPte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);

      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
         tmpPte = Pte & ~PA_DIRTY;
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));

      if (Pte & PA_DIRTY)
      {
         MiFlushTlb((PULONG)Pt, Address);
      }
      else
      {
         MmUnmapPageTable((PULONG)Pt);
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);

      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
      } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));

      if (Pte & PA_DIRTY)
      {
         MiFlushTlb(Pt, Address);
      }
      else
      {
         MmUnmapPageTable(Pt);
      }
   }
}

VOID
NTAPI
MmSetDirtyPage(PEPROCESS Process, PVOID Address)
{
   if (Address < MmSystemRangeStart && Process == NULL)
   {
      DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
      ASSERT(FALSE);
   }
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG Pte;
      ULONGLONG tmpPte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
         tmpPte = Pte | PA_DIRTY;
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
      if (!(Pte & PA_DIRTY))
      {
         MiFlushTlb((PULONG)Pt, Address);
      }
      else
      {
         MmUnmapPageTable((PULONG)Pt);
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
      } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
      if (!(Pte & PA_DIRTY))
      {
         MiFlushTlb(Pt, Address);
      }
      else
      {
         MmUnmapPageTable(Pt);
      }
   }
}

VOID
NTAPI
MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
{
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG Pte;
      ULONGLONG tmpPte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
         tmpPte = Pte | PA_PRESENT;
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
      if (!(Pte & PA_PRESENT))
      {
         MiFlushTlb((PULONG)Pt, Address);
      }
      else
      {
         MmUnmapPageTable((PULONG)Pt);
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }

      do
      {
         Pte = *Pt;
      } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
      if (!(Pte & PA_PRESENT))
      {
         MiFlushTlb(Pt, Address);
      }
      else
      {
         MmUnmapPageTable(Pt);
      }
   }
}

BOOLEAN
NTAPI
MmIsPagePresent(PEPROCESS Process, PVOID Address)
{
   if (Ke386Pae)
   {
      return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
   }
   else
   {
      return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
   }
}

BOOLEAN
NTAPI
MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
{
   if (Ke386Pae)
   {
      ULONGLONG Entry;
      Entry = MmGetPageEntryForProcessForPAE(Process, Address);
      return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
   }
   else
   {
      ULONG Entry;
      Entry = MmGetPageEntryForProcess(Process, Address);
      return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
   }
}

NTSTATUS
NTAPI
MmCreateVirtualMappingForKernel(PVOID Address,
                                ULONG flProtect,
                                PPFN_NUMBER Pages,
                                ULONG PageCount)
{
   ULONG Attributes;
   ULONG i;
   PVOID Addr;
   ULONG PdeOffset, oldPdeOffset;
   BOOLEAN NoExecute = FALSE;

   DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
           Address, flProtect, Pages, PageCount);

   if (Address < MmSystemRangeStart)
   {
      DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
      ASSERT(FALSE);
   }

   Attributes = ProtectToPTE(flProtect);
   if (Attributes & 0x80000000)
   {
      NoExecute = TRUE;
   }
   Attributes &= 0xfff;
   if (Ke386GlobalPagesEnabled)
   {
      Attributes |= PA_GLOBAL;
   }

   Addr = Address;

   if (Ke386Pae)
   {
      PULONGLONG Pt = NULL;
      ULONGLONG Pte;

      oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
      for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
      {
         if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
         {
            DPRINT1("Setting physical address but not allowing access at address "
                    "0x%.8X with attributes %x/%x.\n",
                    Addr, Attributes, flProtect);
            ASSERT(FALSE);
         }

         PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
         if (oldPdeOffset != PdeOffset)
         {
            Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
            if (Pt == NULL)
            {
               ASSERT(FALSE);
            }
         }
         else
         {
            Pt++;
         }
         oldPdeOffset = PdeOffset;

         Pte = PFN_TO_PTE(Pages[i]) | Attributes;
         if (NoExecute)
         {
            Pte |= 0x8000000000000000LL;
         }
         Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
         if (Pte != 0LL)
         {
            ASSERT(FALSE);
         }
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
      Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      Pt--;

      for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
      {
         if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
         {
            DPRINT1("Setting physical address but not allowing access at address "
                    "0x%.8X with attributes %x/%x.\n",
                    Addr, Attributes, flProtect);
            ASSERT(FALSE);
         }

         PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
         if (oldPdeOffset != PdeOffset)
         {
            Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
            if (Pt == NULL)
            {
               ASSERT(FALSE);
            }
         }
         else
         {
            Pt++;
         }
         oldPdeOffset = PdeOffset;

         Pte = *Pt;
         if (Pte != 0)
         {
            ASSERT(FALSE);
         }
         (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
      }
   }

   return(STATUS_SUCCESS);
}

NTSTATUS
NTAPI
MmCreatePageFileMapping(PEPROCESS Process,
                        PVOID Address,
                        SWAPENTRY SwapEntry)
{
   if (Process == NULL && Address < MmSystemRangeStart)
   {
      DPRINT1("No process\n");
      ASSERT(FALSE);
   }
   if (Process != NULL && Address >= MmSystemRangeStart)
   {
      DPRINT1("Setting kernel address with process context\n");
      ASSERT(FALSE);
   }
   if (SwapEntry & (1 << 31))
   {
      ASSERT(FALSE);
   }

   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG Pte;
      ULONGLONG tmpPte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      tmpPte = SwapEntry << 1;
      Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
      if (PAE_PAGE_MASK((Pte)) != 0)
      {
         MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
      }

      if (Pte != 0)
      {
         MiFlushTlb((PULONG)Pt, Address);
      }
      else
      {
         MmUnmapPageTable((PULONG)Pt);
      }
   }
   else
   {
      PULONG Pt;
      ULONG Pte;

      Pt = MmGetPageTableForProcess(Process, Address, TRUE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      Pte = *Pt;
      if (PAGE_MASK((Pte)) != 0)
      {
         MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
      }
      (void)InterlockedExchangeUL(Pt, SwapEntry << 1);
      if (Pte != 0)
      {
         MiFlushTlb(Pt, Address);
      }
      else
      {
         MmUnmapPageTable(Pt);
      }
   }
   if (Process != NULL &&
       ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
       Address < MmSystemRangeStart)
   {
     PUSHORT Ptrc;
     ULONG Idx;

     Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
     Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
     Ptrc[Idx]++;
   }
   return(STATUS_SUCCESS);
}


NTSTATUS
NTAPI
MmCreateVirtualMappingUnsafe(PEPROCESS Process,
                             PVOID Address,
                             ULONG flProtect,
                             PPFN_NUMBER Pages,
                             ULONG PageCount)
{
   ULONG Attributes;
   PVOID Addr;
   ULONG i;
   ULONG oldPdeOffset, PdeOffset;
   BOOLEAN NoExecute = FALSE;

   DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
          Process, Address, flProtect, Pages, *Pages, PageCount);

   if (Process == NULL)
   {
      if (Address < MmSystemRangeStart)
      {
         DPRINT1("No process\n");
         ASSERT(FALSE);
      }
      if (PageCount > 0x10000 ||
          (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
      {
         DPRINT1("Page count to large\n");
         ASSERT(FALSE);
      }
   }
   else
   {
      if (Address >= MmSystemRangeStart)
      {
         DPRINT1("Setting kernel address with process context\n");
         ASSERT(FALSE);
      }
      if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
          (ULONG_PTR) Address / PAGE_SIZE + PageCount >
          (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
      {
         DPRINT1("Page Count to large\n");
         ASSERT(FALSE);
      }
   }

   Attributes = ProtectToPTE(flProtect);
   if (Attributes & 0x80000000)
   {
      NoExecute = TRUE;
   }
   Attributes &= 0xfff;
   if (Address >= MmSystemRangeStart)
   {
      Attributes &= ~PA_USER;
      if (Ke386GlobalPagesEnabled)
      {
         Attributes |= PA_GLOBAL;
      }
   }
   else
   {
      Attributes |= PA_USER;
   }

   Addr = Address;

   if (Ke386Pae)
   {
      ULONGLONG Pte, tmpPte;
      PULONGLONG Pt = NULL;

      oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
      for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
      {
         if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
         {
            DPRINT1("Setting physical address but not allowing access at address "
                    "0x%.8X with attributes %x/%x.\n",
                    Addr, Attributes, flProtect);
            ASSERT(FALSE);
         }
         PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
         if (oldPdeOffset != PdeOffset)
         {
            MmUnmapPageTable((PULONG)Pt);
            Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
            if (Pt == NULL)
            {
               ASSERT(FALSE);
            }
         }
         else
         {
            Pt++;
         }
         oldPdeOffset = PdeOffset;

         MmMarkPageMapped(Pages[i]);
         tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
         if (NoExecute)
         {
            tmpPte |= 0x8000000000000000LL;
         }
         Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
         if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
         {
            ASSERT(FALSE);
         }
         if (PAE_PAGE_MASK((Pte)) != 0LL)
         {
            MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
         }
         if (Address < MmSystemRangeStart &&
             ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
             Attributes & PA_PRESENT)
         {
            PUSHORT Ptrc;

            Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;

            Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
         }
         if (Pte != 0LL)
         {
            if (Address > MmSystemRangeStart ||
                (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
            {
              MiFlushTlb((PULONG)Pt, Address);
            }
         }
      }
      if (Addr > Address)
      {
         MmUnmapPageTable((PULONG)Pt);
      }
   }
   else
   {
      PULONG Pt = NULL;
      ULONG Pte;
      oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
      for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
      {
         if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
         {
            DPRINT1("Setting physical address but not allowing access at address "
                    "0x%.8X with attributes %x/%x.\n",
                    Addr, Attributes, flProtect);
            ASSERT(FALSE);
         }
         PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
         if (oldPdeOffset != PdeOffset)
         {
            MmUnmapPageTable(Pt);
            Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
            if (Pt == NULL)
            {
               ASSERT(FALSE);
            }
         }
         else
         {
            Pt++;
         }
         oldPdeOffset = PdeOffset;

         Pte = *Pt;
         MmMarkPageMapped(Pages[i]);
         if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
         {
            ASSERT(FALSE);
         }
         if (PAGE_MASK((Pte)) != 0)
         {
            MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
         }
         (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
         if (Address < MmSystemRangeStart &&
             ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
             Attributes & PA_PRESENT)
         {
            PUSHORT Ptrc;

            Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;

            Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
         }
         if (Pte != 0)
         {
            if (Address > MmSystemRangeStart ||
                (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
            {
               MiFlushTlb(Pt, Address);
            }
         }
      }
      if (Addr > Address)
      {
         MmUnmapPageTable(Pt);
      }
   }
   return(STATUS_SUCCESS);
}

NTSTATUS
NTAPI
MmCreateVirtualMapping(PEPROCESS Process,
                       PVOID Address,
                       ULONG flProtect,
                       PPFN_NUMBER Pages,
                       ULONG PageCount)
{
   ULONG i;

   for (i = 0; i < PageCount; i++)
   {
      if (!MmIsPageInUse(Pages[i]))
      {
         DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i]));
         ASSERT(FALSE);
      }
   }

   return(MmCreateVirtualMappingUnsafe(Process,
                                       Address,
                                       flProtect,
                                       Pages,
                                       PageCount));
}

ULONG
NTAPI
MmGetPageProtect(PEPROCESS Process, PVOID Address)
{
   ULONG Entry;
   ULONG Protect;
   if (Ke386Pae)
   {
      Entry = MmGetPageEntryForProcessForPAE(Process, Address);
   }
   else
   {
      Entry = MmGetPageEntryForProcess(Process, Address);
   }

   if (!(Entry & PA_PRESENT))
   {
      Protect = PAGE_NOACCESS;
   }
   else
   {
      if (Entry & PA_READWRITE)
      {
         Protect = PAGE_READWRITE;
      }
      else
      {
         Protect = PAGE_EXECUTE_READ;
      }
      if (Entry & PA_CD)
      {
         Protect |= PAGE_NOCACHE;
      }
      if (Entry & PA_WT)
      {
         Protect |= PAGE_WRITETHROUGH;
      }
      if (!(Entry & PA_USER))
      {
         Protect |= PAGE_SYSTEM;
      }

   }
   return(Protect);
}

VOID
NTAPI
MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
{
   ULONG Attributes = 0;
   BOOLEAN NoExecute = FALSE;

   DPRINT("MmSetPageProtect(Process %x  Address %x  flProtect %x)\n",
          Process, Address, flProtect);

   Attributes = ProtectToPTE(flProtect);
   if (Attributes & 0x80000000)
   {
      NoExecute = TRUE;
   }
   Attributes &= 0xfff;
   if (Address >= MmSystemRangeStart)
   {
      Attributes &= ~PA_USER;
      if (Ke386GlobalPagesEnabled)
      {
         Attributes |= PA_GLOBAL;
      }
   }
   else
   {
      Attributes |= PA_USER;
   }
   if (Ke386Pae)
   {
      PULONGLONG Pt;
      ULONGLONG tmpPte, Pte;

      Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
      if (Pt == NULL)
      {
         DPRINT1("Address %x\n", Address);
         ASSERT(FALSE);
      }
      do
      {
        Pte = *Pt;
        tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
        if (NoExecute)
        {
           tmpPte |= 0x8000000000000000LL;
        }
        else
        {
           tmpPte &= ~0x8000000000000000LL;
        }
      } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));

      MiFlushTlb((PULONG)Pt, Address);
   }
   else
   {
      PULONG Pt;

      Pt = MmGetPageTableForProcess(Process, Address, FALSE);
      if (Pt == NULL)
      {
         ASSERT(FALSE);
      }
      InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
      MiFlushTlb(Pt, Address);
   }
}

/*
 * @implemented
 */
PHYSICAL_ADDRESS NTAPI
MmGetPhysicalAddress(PVOID vaddr)
/*
 * FUNCTION: Returns the physical address corresponding to a virtual address
 */
{
   PHYSICAL_ADDRESS p;

   DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
   if (Ke386Pae)
   {
      ULONGLONG Pte;
      Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
      if (Pte != 0 && Pte & PA_PRESENT)
      {
         p.QuadPart = PAE_PAGE_MASK(Pte);
         p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
      }
      else
      {
         p.QuadPart = 0;
      }
   }
   else
   {
      ULONG Pte;
      Pte = MmGetPageEntryForProcess(NULL, vaddr);
      if (Pte != 0 && Pte & PA_PRESENT)
      {
         p.QuadPart = PAGE_MASK(Pte);
         p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
      }
      else
      {
         p.QuadPart = 0;
      }
   }
   return p;
}

PVOID
NTAPI
MmCreateHyperspaceMapping(PFN_NUMBER Page)
{
   PVOID Address;
   ULONG i;

   if (Ke386Pae)
   {
      ULONGLONG Entry;
      ULONGLONG ZeroEntry = 0LL;
      PULONGLONG Pte;

      Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
      Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;

      if (Page & 1024)
      {
         for (i = Page %1024; i < 1024; i++, Pte++)
         {
            if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
            {
               break;
            }
         }
         if (i >= 1024)
         {
            Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
            for (i = 0; i < Page % 1024; i++, Pte++)
            {
               if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
               {
                  break;
               }
            }
            if (i >= Page % 1024)
            {
               ASSERT(FALSE);
            }
         }
      }
      else
      {
         for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
         {
            if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
            {
               break;
            }
         }
         if ((LONG)i < 0)
         {
            Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
            for (i = 1023; i > Page % 1024; i--, Pte--)
            {
               if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
               {
                  break;
               }
            }
            if (i <= Page % 1024)
            {
               ASSERT(FALSE);
            }
         }
      }
   }
   else
   {
      ULONG Entry;
      PULONG Pte;
      Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
      Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
      if (Page & 1024)
      {
         for (i = Page % 1024; i < 1024; i++, Pte++)
         {
            if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
            {
               break;
            }
         }
         if (i >= 1024)
         {
            Pte = ADDR_TO_PTE(HYPERSPACE);
            for (i = 0; i < Page % 1024; i++, Pte++)
            {
               if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
               {
                  break;
               }
            }
            if (i >= Page % 1024)
            {
               ASSERT(FALSE);
            }
         }
      }
      else
      {
         for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
         {
            if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
            {
               break;
            }
         }
         if ((LONG)i < 0)
         {
            Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
            for (i = 1023; i > Page % 1024; i--, Pte--)
            {
               if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
               {
                  break;
               }
            }
            if (i <= Page % 1024)
            {
               ASSERT(FALSE);
            }
         }
      }
   }
   Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
   __invlpg(Address);
   return Address;
}

PFN_NUMBER
NTAPI
MmChangeHyperspaceMapping(PVOID Address, PFN_NUMBER NewPage)
{
   PFN_NUMBER Pfn;
   ASSERT (IS_HYPERSPACE(Address));
   if (Ke386Pae)
   {
      ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
      Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
      Pfn = PAE_PTE_TO_PFN(Entry);
   }
   else
   {
      ULONG Entry;
      Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
      Pfn = PTE_TO_PFN(Entry);
   }
   __invlpg(Address);
   return Pfn;
}

PFN_NUMBER
NTAPI
MmDeleteHyperspaceMapping(PVOID Address)
{
   PFN_NUMBER Pfn;
   ASSERT (IS_HYPERSPACE(Address));
   if (Ke386Pae)
   {
      ULONGLONG Entry = 0LL;
      Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
      Pfn = PAE_PTE_TO_PFN(Entry);
   }
   else
   {
      ULONG Entry;
      Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
      Pfn = PTE_TO_PFN(Entry);
   }
   __invlpg(Address);
   return Pfn;
}

VOID
NTAPI
MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
{
   ULONG StartOffset, EndOffset, Offset;

   if (Address < MmSystemRangeStart)
   {
      ASSERT(FALSE);
   }
   if (Ke386Pae)
   {
      PULONGLONG PageDirTable;
      PULONGLONG Pde;
      ULONGLONG ZeroPde = 0LL;
      ULONG i;

      for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++)
      {
         if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
         {
            StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
         }
         else
         {
            StartOffset = 0;
         }
         if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)))
         {
            EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
         }
         else
         {
            EndOffset = 511;
         }

         if (Process != NULL && Process != PsGetCurrentProcess())
         {
            PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
            Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
            MmDeleteHyperspaceMapping(PageDirTable);
         }
         else
         {
            Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
         }

         for (Offset = StartOffset; Offset <= EndOffset; Offset++)
         {
            if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
            {
               (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
            }
         }
         MmUnmapPageTable((PULONG)Pde);
      }
   }
   else
   {
      PULONG Pde;
      StartOffset = ADDR_TO_PDE_OFFSET(Address);
      EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size));

      if (Process != NULL && Process != PsGetCurrentProcess())
      {
         Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
      }
      else
      {
         Pde = (PULONG)PAGEDIRECTORY_MAP;
      }
      for (Offset = StartOffset; Offset <= EndOffset; Offset++)
      {
         if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
         {
            (void)InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
         }
      }
      if (Pde != (PULONG)PAGEDIRECTORY_MAP)
      {
         MmDeleteHyperspaceMapping(Pde);
      }
   }
}

VOID
INIT_FUNCTION
NTAPI
MmInitGlobalKernelPageDirectory(VOID)
{
   ULONG i;

   DPRINT("MmInitGlobalKernelPageDirectory()\n");

   if (Ke386Pae)
   {
      PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
      for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
      {
         if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
             !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
             0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
         {
            (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
            if (Ke386GlobalPagesEnabled)
            {
               MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
               CurrentPageDirectory[i] |= PA_GLOBAL;
            }
         }
      }
   }
   else
   {
      PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
      for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
      {
         if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
             i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
             0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
         {
            MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
            if (Ke386GlobalPagesEnabled)
            {
               MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
               CurrentPageDirectory[i] |= PA_GLOBAL;
            }
         }
      }
   }
}

ULONG
NTAPI
MiGetUserPageDirectoryCount(VOID)
{
   return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart);
}

VOID
INIT_FUNCTION
NTAPI
MiInitPageDirectoryMap(VOID)
{
   MEMORY_AREA* kernel_map_desc = NULL;
   MEMORY_AREA* hyperspace_desc = NULL;
   PHYSICAL_ADDRESS BoundaryAddressMultiple;
   PVOID BaseAddress;
   NTSTATUS Status;

   DPRINT("MiInitPageDirectoryMap()\n");

   BoundaryAddressMultiple.QuadPart = 0;
   BaseAddress = (PVOID)PAGETABLE_MAP;
   Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
                               MEMORY_AREA_SYSTEM,
                               &BaseAddress,
                               Ke386Pae ? 0x800000 : 0x400000,
                               PAGE_READWRITE,
                               &kernel_map_desc,
                               TRUE,
                               0,
                               BoundaryAddressMultiple);
   if (!NT_SUCCESS(Status))
   {
      ASSERT(FALSE);
   }
   BaseAddress = (PVOID)HYPERSPACE;
   Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
                               MEMORY_AREA_SYSTEM,
                               &BaseAddress,
                               0x400000,
                               PAGE_READWRITE,
                               &hyperspace_desc,
                               TRUE,
                               0,
                               BoundaryAddressMultiple);
   if (!NT_SUCCESS(Status))
   {
      ASSERT(FALSE);
   }
}

/* EOF */