ReactOS  0.4.11-dev-433-g473ca91
kiinit.c
Go to the documentation of this file.
1 /*
2  * PROJECT: ReactOS Kernel
3  * LICENSE: GPL - See COPYING in the top level directory
4  * FILE: ntoskrnl/ke/i386/kiinit.c
5  * PURPOSE: Kernel Initialization for x86 CPUs
6  * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7  */
8 
9 /* INCLUDES *****************************************************************/
10 
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14 #include "internal/i386/trap_x.h"
15 
16 /* GLOBALS *******************************************************************/
17 
18 /* Boot and double-fault/NMI/DPC stack */
20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
23 
24 /* Spinlocks used only on X86 */
27 
28 /* Perf */
31 
32 /* FUNCTIONS *****************************************************************/
33 
35 VOID
36 NTAPI
38 {
39  ULONG CpuCount;
40  BOOLEAN FbCaching = FALSE;
43  ULONG i, Affinity, Sample = 0;
44  PFX_SAVE_AREA FxSaveArea;
45  ULONG MXCsrMask = 0xFFBF;
46  CPU_INFO CpuInfo;
47  KI_SAMPLE_MAP Samples[10];
48  PKI_SAMPLE_MAP CurrentSample = Samples;
49  LARGE_IDENTITY_MAP IdentityMap;
50 
51  /* Check for large page support */
53  {
54  /* Do an IPI to enable it on all CPUs */
57 
58  /* Free the pages allocated for identity map */
59  Ki386FreeIdentityMap(&IdentityMap);
60  }
61 
62  /* Check for global page support */
64  {
65  /* Do an IPI to enable it on all CPUs */
66  CpuCount = KeNumberProcessors;
68  }
69 
70  /* Check for PAT and/or MTRR support */
71  if (KeFeatureBits & (KF_PAT | KF_MTRR))
72  {
73  /* Query the HAL to make sure we can use it */
75  sizeof(BOOLEAN),
76  &FbCaching,
77  &ReturnLength);
78  if ((NT_SUCCESS(Status)) && (FbCaching))
79  {
80  /* We can't, disable it */
82  }
83  }
84 
85  /* Check for PAT support and enable it */
87 
88  /* Assume no errata for now */
89  SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
90 
91  /* Check if we have an NPX */
92  if (KeI386NpxPresent)
93  {
94  /* Loop every CPU */
96  for (Affinity = 1; i; Affinity <<= 1)
97  {
98  /* Check if this is part of the set */
99  if (i & Affinity)
100  {
101  /* Run on this CPU */
102  i &= ~Affinity;
103  KeSetSystemAffinityThread(Affinity);
104 
105  /* Detect FPU errata */
106  if (KiIsNpxErrataPresent())
107  {
108  /* Disable NPX support */
111  ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
112  TRUE;
113  break;
114  }
115  }
116  }
117  }
118 
119  /* If there's no NPX, then we're emulating the FPU */
120  SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
122 
123  /* Check if there's no NPX, so that we can disable associated features */
124  if (!KeI386NpxPresent)
125  {
126  /* Remove NPX-related bits */
128 
129  /* Disable kernel flags */
131 
132  /* Disable processor features that might've been set until now */
135  SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
136  SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
137  SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
138  }
139 
140  /* Check for CR4 support */
141  if (KeFeatureBits & KF_CR4)
142  {
143  /* Do an IPI call to enable the Debug Exceptions */
144  CpuCount = KeNumberProcessors;
146  }
147 
148  /* Check if FXSR was found */
149  if (KeFeatureBits & KF_FXSR)
150  {
151  /* Do an IPI call to enable the FXSR */
152  CpuCount = KeNumberProcessors;
154 
155  /* Check if XMM was found too */
156  if (KeFeatureBits & KF_XMMI)
157  {
158  /* Do an IPI call to enable XMMI exceptions */
159  CpuCount = KeNumberProcessors;
161 
162  /* FIXME: Implement and enable XMM Page Zeroing for Mm */
163 
164  /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
165  *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP
166  }
167  }
168 
169  /* Check for, and enable SYSENTER support */
171 
172  /* Loop every CPU */
173  i = KeActiveProcessors;
174  for (Affinity = 1; i; Affinity <<= 1)
175  {
176  /* Check if this is part of the set */
177  if (i & Affinity)
178  {
179  /* Run on this CPU */
180  i &= ~Affinity;
181  KeSetSystemAffinityThread(Affinity);
182 
183  /* Reset MHz to 0 for this CPU */
184  KeGetCurrentPrcb()->MHz = 0;
185 
186  /* Check if we can use RDTSC */
187  if (KeFeatureBits & KF_RDTSC)
188  {
189  /* Start sampling loop */
190  for (;;)
191  {
192  /* Do a dummy CPUID to start the sample */
193  KiCpuId(&CpuInfo, 0);
194 
195  /* Fill out the starting data */
196  CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
197  CurrentSample->TSCStart = __rdtsc();
198  CurrentSample->PerfFreq.QuadPart = -50000;
199 
200  /* Sleep for this sample */
202  FALSE,
203  &CurrentSample->PerfFreq);
204 
205  /* Do another dummy CPUID */
206  KiCpuId(&CpuInfo, 0);
207 
208  /* Fill out the ending data */
209  CurrentSample->PerfEnd =
210  KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
211  CurrentSample->TSCEnd = __rdtsc();
212 
213  /* Calculate the differences */
214  CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
215  CurrentSample->PerfStart.QuadPart;
216  CurrentSample->TSCDelta = CurrentSample->TSCEnd -
217  CurrentSample->TSCStart;
218 
219  /* Compute CPU Speed */
220  CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
221  CurrentSample->
222  PerfFreq.QuadPart + 500000) /
223  (CurrentSample->PerfDelta *
224  1000000));
225 
226  /* Check if this isn't the first sample */
227  if (Sample)
228  {
229  /* Check if we got a good precision within 1MHz */
230  if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
231  (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
232  (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
233  {
234  /* We did, stop sampling */
235  break;
236  }
237  }
238 
239  /* Move on */
240  CurrentSample++;
241  Sample++;
242 
243  if (Sample == RTL_NUMBER_OF(Samples))
244  {
245  /* No luck. Average the samples and be done */
246  ULONG TotalMHz = 0;
247  while (Sample--)
248  {
249  TotalMHz += Samples[Sample].MHz;
250  }
251  CurrentSample[-1].MHz = TotalMHz / RTL_NUMBER_OF(Samples);
252  DPRINT1("Sampling CPU frequency failed. Using average of %lu MHz\n", CurrentSample[-1].MHz);
253  break;
254  }
255  }
256 
257  /* Save the CPU Speed */
258  KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
259  }
260 
261  /* Check if we have MTRR */
262  if (KeFeatureBits & KF_MTRR)
263  {
264  /* Then manually initialize MTRR for the CPU */
265  KiInitializeMTRR(i ? FALSE : TRUE);
266  }
267 
268  /* Check if we have AMD MTRR and initialize it for the CPU */
270 
271  /* Check if this is a buggy Pentium and apply the fixup if so */
273 
274  /* Check if the CPU supports FXSR */
275  if (KeFeatureBits & KF_FXSR)
276  {
277  /* Get the current thread NPX state */
278  FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
279 
280  /* Clear initial MXCsr mask */
281  FxSaveArea->U.FxArea.MXCsrMask = 0;
282 
283  /* Save the current NPX State */
284  Ke386SaveFpuState(FxSaveArea);
285 
286  /* Check if the current mask doesn't match the reserved bits */
287  if (FxSaveArea->U.FxArea.MXCsrMask != 0)
288  {
289  /* Then use whatever it's holding */
290  MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
291  }
292 
293  /* Check if nobody set the kernel-wide mask */
294  if (!KiMXCsrMask)
295  {
296  /* Then use the one we calculated above */
297  KiMXCsrMask = MXCsrMask;
298  }
299  else
300  {
301  /* Was it set to the same value we found now? */
302  if (KiMXCsrMask != MXCsrMask)
303  {
304  /* No, something is definitely wrong */
305  KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
306  KF_FXSR,
307  KiMXCsrMask,
308  MXCsrMask,
309  0);
310  }
311  }
312 
313  /* Now set the kernel mask */
314  KiMXCsrMask &= MXCsrMask;
315  }
316  }
317  }
318 
319  /* Return affinity back to where it was */
321 
322  /* NT allows limiting the duration of an ISR with a registry key */
324  {
325  /* FIXME: TODO */
326  DPRINT1("ISR Time Limit not yet supported\n");
327  }
328 
329  /* Set CR0 features based on detected CPU */
330  KiSetCR0Bits();
331 }
332 
334 VOID
335 NTAPI
336 KiInitializePcr(IN ULONG ProcessorNumber,
337  IN PKIPCR Pcr,
338  IN PKIDTENTRY Idt,
339  IN PKGDTENTRY Gdt,
340  IN PKTSS Tss,
341  IN PKTHREAD IdleThread,
342  IN PVOID DpcStack)
343 {
344  /* Setup the TIB */
345  Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
346  Pcr->NtTib.StackBase = 0;
347  Pcr->NtTib.StackLimit = 0;
348  Pcr->NtTib.Self = NULL;
349 
350  /* Set the Current Thread */
351  Pcr->PrcbData.CurrentThread = IdleThread;
352 
353  /* Set pointers to ourselves */
354  Pcr->SelfPcr = (PKPCR)Pcr;
355  Pcr->Prcb = &Pcr->PrcbData;
356 
357  /* Set the PCR Version */
358  Pcr->MajorVersion = PCR_MAJOR_VERSION;
359  Pcr->MinorVersion = PCR_MINOR_VERSION;
360 
361  /* Set the PCRB Version */
362  Pcr->PrcbData.MajorVersion = 1;
363  Pcr->PrcbData.MinorVersion = 1;
364 
365  /* Set the Build Type */
366  Pcr->PrcbData.BuildType = 0;
367 #ifndef CONFIG_SMP
368  Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
369 #endif
370 #if DBG
371  Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
372 #endif
373 
374  /* Set the Processor Number and current Processor Mask */
375  Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
376  Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
377 
378  /* Set the PRCB for this Processor */
379  KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
380 
381  /* Start us out at PASSIVE_LEVEL */
382  Pcr->Irql = PASSIVE_LEVEL;
383 
384  /* Set the GDI, IDT, TSS and DPC Stack */
385  Pcr->GDT = (PVOID)Gdt;
386  Pcr->IDT = Idt;
387  Pcr->TSS = Tss;
388  Pcr->TssCopy = Tss;
389  Pcr->PrcbData.DpcStack = DpcStack;
390 
391  /* Setup the processor set */
392  Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
393 }
394 
396 VOID
397 NTAPI
399  IN PKTHREAD InitThread,
400  IN PVOID IdleStack,
401  IN PKPRCB Prcb,
402  IN CCHAR Number,
403  IN PLOADER_PARAMETER_BLOCK LoaderBlock)
404 {
405  BOOLEAN NpxPresent;
406  ULONG FeatureBits;
407  ULONG PageDirectory[2];
408  PVOID DpcStack;
409  ULONG Vendor[3];
410  KIRQL DummyIrql;
411 
412  /* Detect and set the CPU Type */
414 
415  /* Check if an FPU is present */
416  NpxPresent = KiIsNpxPresent();
417 
418  /* Initialize the Power Management Support for this PRCB */
419  PoInitializePrcb(Prcb);
420 
421  /* Bugcheck if this is a 386 CPU */
422  if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
423 
424  /* Get the processor features for the CPU */
425  FeatureBits = KiGetFeatureBits();
426 
427  /* Set the default NX policy (opt-in) */
428  SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
429 
430  /* Check if NPX is always on */
431  if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
432  {
433  /* Set it always on */
434  SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
435  FeatureBits |= KF_NX_ENABLED;
436  }
437  else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
438  {
439  /* Set it in opt-out mode */
440  SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
441  FeatureBits |= KF_NX_ENABLED;
442  }
443  else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
444  (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
445  {
446  /* Set the feature bits */
447  FeatureBits |= KF_NX_ENABLED;
448  }
449  else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
450  (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
451  {
452  /* Set disabled mode */
453  SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
454  FeatureBits |= KF_NX_DISABLED;
455  }
456 
457  /* Save feature bits */
458  Prcb->FeatureBits = FeatureBits;
459 
460  /* Save CPU state */
461  KiSaveProcessorControlState(&Prcb->ProcessorState);
462 
463  /* Get cache line information for this CPU */
465 
466  /* Initialize spinlocks and DPC data */
467  KiInitSpinLocks(Prcb, Number);
468 
469  /* Check if this is the Boot CPU */
470  if (!Number)
471  {
472  /* Set Node Data */
473  KeNodeBlock[0] = &KiNode0;
474  Prcb->ParentNode = KeNodeBlock[0];
475  KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
476 
477  /* Set boot-level flags */
478  KeI386NpxPresent = NpxPresent;
479  KeI386CpuType = Prcb->CpuType;
480  KeI386CpuStep = Prcb->CpuStep;
482  KeProcessorLevel = (USHORT)Prcb->CpuType;
483  if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
484  KeFeatureBits = FeatureBits;
487 
488  /* Detect 8-byte compare exchange support */
489  if (!(KeFeatureBits & KF_CMPXCHG8B))
490  {
491  /* Copy the vendor string */
492  RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
493 
494  /* Bugcheck the system. Windows *requires* this */
495  KeBugCheckEx(UNSUPPORTED_PROCESSOR,
496  (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
497  Vendor[0],
498  Vendor[1],
499  Vendor[2]);
500  }
501 
502  /* Set the current MP Master KPRCB to the Boot PRCB */
503  Prcb->MultiThreadSetMaster = Prcb;
504 
505  /* Lower to APC_LEVEL */
507 
508  /* Initialize some spinlocks */
511 
512  /* Initialize portable parts of the OS */
513  KiInitSystem();
514 
515  /* Initialize the Idle Process and the Process Listhead */
517  PageDirectory[0] = 0;
518  PageDirectory[1] = 0;
519  KeInitializeProcess(InitProcess,
520  0,
521  0xFFFFFFFF,
522  PageDirectory,
523  FALSE);
524  InitProcess->QuantumReset = MAXCHAR;
525  }
526  else
527  {
528  /* FIXME */
529  DPRINT1("SMP Boot support not yet present\n");
530  }
531 
532  /* Setup the Idle Thread */
533  KeInitializeThread(InitProcess,
534  InitThread,
535  NULL,
536  NULL,
537  NULL,
538  NULL,
539  NULL,
540  IdleStack);
541  InitThread->NextProcessor = Number;
542  InitThread->Priority = HIGH_PRIORITY;
543  InitThread->State = Running;
544  InitThread->Affinity = 1 << Number;
545  InitThread->WaitIrql = DISPATCH_LEVEL;
546  InitProcess->ActiveProcessors = 1 << Number;
547 
548  /* HACK for MmUpdatePageDir */
549  ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
550 
551  /* Set basic CPU Features that user mode can read */
552  SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
554  SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
556  SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
560  SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
562  SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
564 
565  /* Set up the thread-related fields in the PRCB */
566  Prcb->CurrentThread = InitThread;
567  Prcb->NextThread = NULL;
568  Prcb->IdleThread = InitThread;
569 
570  /* Initialize the Kernel Executive */
571  ExpInitializeExecutive(Number, LoaderBlock);
572 
573  /* Only do this on the boot CPU */
574  if (!Number)
575  {
576  /* Calculate the time reciprocal */
580 
581  /* Update DPC Values in case they got updated by the executive */
582  Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
583  Prcb->MinimumDpcRate = KiMinimumDpcRate;
584  Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
585 
586  /* Allocate the DPC Stack */
587  DpcStack = MmCreateKernelStack(FALSE, 0);
588  if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
589  Prcb->DpcStack = DpcStack;
590 
591  /* Allocate the IOPM save area. */
593  PAGE_SIZE * 2,
594  ' eK');
595  if (!Ki386IopmSaveArea)
596  {
597  /* Bugcheck. We need this for V86/VDM support. */
598  KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
599  }
600  }
601 
602  /* Raise to Dispatch */
604  &DummyIrql);
605 
606  /* Set the Idle Priority to 0. This will jump into Phase 1 */
607  KeSetPriorityThread(InitThread, 0);
608 
609  /* If there's no thread scheduled, put this CPU in the Idle summary */
610  KiAcquirePrcbLock(Prcb);
611  if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
612  KiReleasePrcbLock(Prcb);
613 
614  /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
616  &DummyIrql);
617  LoaderBlock->Prcb = 0;
618 }
619 
621 VOID
622 FASTCALL
624  IN PKIDTENTRY *Idt,
625  IN PKIPCR *Pcr,
626  IN PKTSS *Tss)
627 {
628  KDESCRIPTOR GdtDescriptor, IdtDescriptor;
629  KGDTENTRY TssSelector, PcrSelector;
630  USHORT Tr, Fs;
631 
632  /* Get GDT and IDT descriptors */
633  Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
634  __sidt(&IdtDescriptor.Limit);
635 
636  /* Save IDT and GDT */
637  *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
638  *Idt = (PKIDTENTRY)IdtDescriptor.Base;
639 
640  /* Get TSS and FS Selectors */
641  Tr = Ke386GetTr();
642  Fs = Ke386GetFs();
643 
644  /* Get PCR Selector, mask it and get its GDT Entry */
645  PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
646 
647  /* Get the KPCR itself */
648  *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
649  PcrSelector.HighWord.Bytes.BaseMid << 16 |
650  PcrSelector.HighWord.Bytes.BaseHi << 24);
651 
652  /* Get TSS Selector, mask it and get its GDT Entry */
653  TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
654 
655  /* Get the KTSS itself */
656  *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
657  TssSelector.HighWord.Bytes.BaseMid << 16 |
658  TssSelector.HighWord.Bytes.BaseHi << 24);
659 }
660 
662 VOID
663 NTAPI
665 {
667 
668  /* Initialize the kernel for the current CPU */
673  KeNumberProcessors - 1,
674  KeLoaderBlock);
675 
676  /* Set the priority of this thread to 0 */
677  Thread = KeGetCurrentThread();
678  Thread->Priority = 0;
679 
680  /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
681  _enable();
683 
684  /* Set the right wait IRQL */
685  Thread->WaitIrql = DISPATCH_LEVEL;
686 
687  /* Jump into the idle loop */
688  KiIdleLoop();
689 }
690 
691 static
692 VOID
694  PVOID Address)
695 {
696  PHARDWARE_PTE PointerPte;
697 
698  /* Make sure the address is page aligned */
699  ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address);
700 
701  /* Get the PTE address */
702  PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE);
703  ASSERT(PointerPte->Valid);
704  ASSERT(PointerPte->Write);
705 
706  /* Set as read-only */
707  PointerPte->Write = 0;
708 
709  /* Flush the TLB entry */
710  __invlpg(Address);
711 }
712 
714 VOID
715 NTAPI
717 {
718  ULONG Cpu;
719  PKTHREAD InitialThread;
720  ULONG InitialStack;
721  PKGDTENTRY Gdt;
722  PKIDTENTRY Idt;
723  KIDTENTRY NmiEntry, DoubleFaultEntry;
724  PKTSS Tss;
725  PKIPCR Pcr;
726  KIRQL DummyIrql;
727 
728  /* Boot cycles timestamp */
729  BootCycles = __rdtsc();
730 
731  /* Save the loader block and get the current CPU */
732  KeLoaderBlock = LoaderBlock;
733  Cpu = KeNumberProcessors;
734  if (!Cpu)
735  {
736  /* If this is the boot CPU, set FS and the CPU Number*/
737  Ke386SetFs(KGDT_R0_PCR);
739 
740  /* Set the initial stack and idle thread as well */
741  LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
742  LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
743  }
744 
745  /* Save the initial thread and stack */
746  InitialStack = LoaderBlock->KernelStack;
747  InitialThread = (PKTHREAD)LoaderBlock->Thread;
748 
749  /* Clean the APC List Head */
750  InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
751 
752  /* Initialize the machine type */
754 
755  /* Skip initial setup if this isn't the Boot CPU */
756  if (Cpu) goto AppCpuInit;
757 
758  /* Get GDT, IDT, PCR and TSS pointers */
759  KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
760 
761  /* Setup the TSS descriptors and entries */
762  Ki386InitializeTss(Tss, Idt, Gdt);
763 
764  /* Initialize the PCR */
765  RtlZeroMemory(Pcr, PAGE_SIZE);
766  KiInitializePcr(Cpu,
767  Pcr,
768  Idt,
769  Gdt,
770  Tss,
771  InitialThread,
773 
774  /* Set us as the current process */
775  InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
776 
777  /* Clear DR6/7 to cleanup bootloader debugging */
781 
782  /* Setup the IDT */
784 
785  /* Load Ring 3 selectors for DS/ES */
786  Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
787  Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
788 
789  /* Save NMI and double fault traps */
790  RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
791  RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
792 
793  /* Copy kernel's trap handlers */
794  RtlCopyMemory(Idt,
796  KiIdtDescriptor.Limit + 1);
797 
798  /* Restore NMI and double fault */
799  RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
800  RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
801 
802 AppCpuInit:
803  /* Loop until we can release the freeze lock */
804  do
805  {
806  /* Loop until execution can continue */
807  while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
809 
810  /* Setup CPU-related fields */
812  __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
815 
816  /* Initialize the Processor with HAL */
818 
819  /* Set active processors */
822 
823  /* Check if this is the boot CPU */
824  if (!Cpu)
825  {
826  /* Initialize debugging system */
828 
829  /* Check for break-in */
831 
832  /* Make the lowest page of the boot and double fault stack read-only */
833  KiMarkPageAsReadOnly(P0BootStackData);
834  KiMarkPageAsReadOnly(KiDoubleFaultStackData);
835  }
836 
837  /* Raise to HIGH_LEVEL */
839  &DummyIrql);
840 
841  /* Switch to new kernel stack and start kernel bootstrapping */
842  KiSwitchToBootStack(InitialStack & ~3);
843 }
ULONG64 Valid
Definition: mmtypes.h:66
VOID NTAPI KiInitMachineDependent(VOID)
Definition: kiinit.c:45
DWORD *typedef PVOID
Definition: winlogon.h:61
VOID NTAPI KiInitSystem(VOID)
Definition: krnlinit.c:71
VOID KiGetCacheInformation()
ULONG KeMaximumIncrement
Definition: clock.c:20
struct _KGDTENTRY::@2324::@2325 Bytes
IN CINT OUT PVOID IN ULONG OUT PULONG ReturnLength
Definition: dumpinfo.c:39
#define IN
Definition: typedefs.h:38
LARGE_INTEGER NTAPI KeQueryPerformanceCounter(IN PLARGE_INTEGER PerformanceFreq)
Definition: timer.c:138
#define KeRaiseIrql(irql, oldIrql)
Definition: env_spec_w32.h:597
#define TRUE
Definition: types.h:120
NTSYSAPI VOID NTAPI RtlCopyMemory(VOID UNALIGNED *Destination, CONST VOID UNALIGNED *Source, ULONG Length)
PVOID ULONG Address
Definition: oprghdlr.h:14
BOOLEAN KiI386PentiumLockErrataPresent
Definition: cpu.c:40
#define KeLowerIrql(oldIrql)
Definition: env_spec_w32.h:602
#define PKGDTENTRY
Definition: ketypes.h:438
FORCEINLINE VOID KiReleasePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:224
LARGE_INTEGER NTAPI KiComputeReciprocal(IN LONG Divisor, OUT PUCHAR Shift)
Definition: krnlinit.c:123
LARGE_INTEGER PerfStart
Definition: ke.h:54
LARGE_INTEGER PerfEnd
Definition: ke.h:55
VOID NTAPI KiSetCR0Bits(VOID)
Definition: cpu.c:728
#define KF_CMPXCHG8B
Definition: ketypes.h:150
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel)?(CompletionRoutine!=NULL):TRUE)
KNODE KiNode0
Definition: krnlinit.c:38
USHORT BaseLow
Definition: ketypes.h:334
char * strstr(char *String1, char *String2)
Definition: utclib.c:653
VOID NTAPI KeRevertToUserAffinityThread(VOID)
Definition: thrdobj.c:1030
#define NX_SUPPORT_POLICY_OPTIN
Definition: ketypes.h:1085
BOOLEAN NTAPI Ki386CreateIdentityMap(IN PLARGE_IDENTITY_MAP IdentityMap, IN PVOID StartPtr, IN ULONG Length)
INIT_SECTION VOID FASTCALL KiGetMachineBootPointers(IN PKGDTENTRY *Gdt, IN PKIDTENTRY *Idt, IN PKIPCR *Pcr, IN PKTSS *Tss)
Definition: kiinit.c:623
LARGE_INTEGER PerfFreq
Definition: ke.h:57
void __cdecl _enable(void)
Definition: intrin_arm.h:373
VOID NTAPI KeInitExceptions(VOID)
Definition: except.c:59
#define KF_RDTSC
Definition: ketypes.h:144
FORCEINLINE struct _KPRCB * KeGetCurrentPrcb(VOID)
Definition: ketypes.h:1062
struct _EPROCESS * PEPROCESS
Definition: nt_native.h:30
struct _KIPCR * PKIPCR
#define PF_COMPARE_EXCHANGE_DOUBLE
SCHAR Priority
Definition: ketypes.h:1672
#define KF_MMX
Definition: ketypes.h:151
ULONG ProcessCount
Definition: kiinit.c:28
PPC_QUAL unsigned long long __rdtsc(void)
Definition: intrin_ppc.h:688
#define KPCR_PRCB_SET_MEMBER
Definition: asm.h:179
VOID NTAPI KiInitializeMTRR(IN BOOLEAN FinalCpu)
Definition: mtrr.c:22
ULONG KeFeatureBits
Definition: krnlinit.c:22
#define HalQuerySystemInformation
Definition: haltypes.h:283
LONGLONG PerfDelta
Definition: ke.h:56
ULONG KeI386NpxPresent
Definition: cpu.c:31
BOOLEAN NTAPI KiIsNpxErrataPresent(VOID)
Definition: cpu.c:1205
ULONG KiAdjustDpcThreshold
Definition: dpc.c:21
KSPIN_LOCK * PKSPIN_LOCK
Definition: env_spec_w32.h:73
#define PF_FLOATING_POINT_EMULATED
ULONG KiTimeLimitIsrMicroseconds
Definition: dpc.c:26
ULONG_PTR NTAPI Ki386EnableGlobalPage(IN ULONG_PTR Context)
Definition: patpge.c:23
ULONGLONG BootCycles
Definition: kiinit.c:30
ULONG_PTR NTAPI Ki386EnableDE(IN ULONG_PTR Context)
Definition: cpu.c:1045
VOID NTAPI KeInitializeProcess(struct _KPROCESS *Process, KPRIORITY Priority, KAFFINITY Affinity, PULONG_PTR DirectoryTableBase, IN BOOLEAN Enable)
#define FASTCALL
Definition: nt_native.h:50
#define PRCB_BUILD_DEBUG
Definition: ketypes.h:242
PVOID Base
Definition: ketypes.h:486
ULONG KeI386FxsrPresent
Definition: cpu.c:31
VOID NTAPI KeSetSystemAffinityThread(IN KAFFINITY Affinity)
Definition: thrdobj.c:1116
#define NX_SUPPORT_POLICY_ALWAYSON
Definition: ketypes.h:1084
VOID NTAPI KiInitializePcr(IN PKIPCR Pcr, IN ULONG ProcessorNumber, IN PKTHREAD IdleThread, IN PVOID DpcStack)
Definition: kiinit.c:89
USHORT Limit
Definition: ketypes.h:485
UCHAR DECLSPEC_ALIGN(16)
Definition: kiinit.c:32
uint32_t ULONG_PTR
Definition: typedefs.h:63
#define KF_LARGE_PAGE
Definition: ketypes.h:148
BOOLEAN NTAPI KiIsNpxPresent(VOID)
Definition: cpu.c:1164
ULONG_PTR P0BootStack
Definition: kiinit.c:34
UCHAR KIRQL
Definition: env_spec_w32.h:591
#define KF_AMDK6MTRR
Definition: ketypes.h:158
ULONG_PTR Thread
Definition: arc.h:499
VOID NTAPI INIT_FUNCTION ExpInitializeExecutive(IN ULONG Cpu, IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Definition: init.c:914
struct _KTHREAD * PKTHREAD
Definition: nt_native.h:28
GLenum GLclampf GLint i
Definition: glfuncs.h:14
#define PF_FLOATING_POINT_PRECISION_ERRATA
NTSTATUS(* NTAPI)(IN PFILE_FULL_EA_INFORMATION EaBuffer, IN ULONG EaLength, OUT PULONG ErrorOffset)
Definition: IoEaTest.cpp:117
#define NX_SUPPORT_POLICY_ALWAYSOFF
Definition: ketypes.h:1083
#define FALSE
Definition: types.h:117
struct _HARDWARE_PTE * PHARDWARE_PTE
VOID NTAPI HalInitializeProcessor(IN ULONG ProcessorNumber, IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Definition: processor.c:48
#define KF_PAT
Definition: ketypes.h:153
#define KF_XMMI
Definition: ketypes.h:156
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:1925
KAPC_STATE ApcState
Definition: ketypes.h:1668
#define NX_SUPPORT_POLICY_OPTOUT
Definition: ketypes.h:1086
BOOLEAN NTAPI KdPollBreakIn(VOID)
Definition: kdmain.c:367
NTSTATUS NTAPI KeDelayExecutionThread(IN KPROCESSOR_MODE WaitMode, IN BOOLEAN Alertable, IN PLARGE_INTEGER Interval OPTIONAL)
Definition: wait.c:283
#define PF_XMMI_INSTRUCTIONS_AVAILABLE
#define RPL_MASK
Definition: ketypes.h:69
FORCEINLINE VOID KeInitializeSpinLock(_Out_ PKSPIN_LOCK SpinLock)
Definition: kefuncs.h:251
FORCEINLINE VOID KiAcquirePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:214
VOID NTAPI KiInitializePAT(VOID)
Definition: patpge.c:61
smooth NULL
Definition: ftsmooth.c:416
union _FX_SAVE_AREA::@2331 U
VOID NTAPI PoInitializePrcb(IN PKPRCB Prcb)
Definition: power.c:389
VOID NTAPI Ki386FreeIdentityMap(IN PLARGE_IDENTITY_MAP IdentityMap)
Definition: ketypes.h:787
#define KGDT_R3_DATA
Definition: ketypes.h:77
PPC_QUAL unsigned long __readfsdword(const unsigned long Offset)
Definition: intrin_ppc.h:382
ULONG KeI386XMMIPresent
Definition: cpu.c:30
LARGE_INTEGER KiTimeIncrementReciprocal
Definition: timerobj.c:18
UCHAR KiTimeIncrementShiftCount
Definition: timerobj.c:19
#define HIGH_PRIORITY
PLOADER_PARAMETER_BLOCK KeLoaderBlock
Definition: krnlinit.c:29
ULONG KiMXCsrMask
Definition: cpu.c:28
ULONG_PTR KiDoubleFaultStack
Definition: kiinit.c:35
#define PCHAR
Definition: match.c:90
FORCEINLINE PFX_SAVE_AREA KiGetThreadNpxArea(IN PKTHREAD Thread)
Definition: ke.h:582
#define InterlockedBitTestAndSet
Definition: interlocked.h:30
VOID FASTCALL Ki386InitializeTss(IN PKTSS Tss, IN PKIDTENTRY Idt, IN PKGDTENTRY Gdt)
Definition: cpu.c:799
INIT_SECTION VOID NTAPI KiSystemStartupBootStack(VOID)
Definition: kiinit.c:664
USHORT KeProcessorArchitecture
Definition: krnlinit.c:19
unsigned char BOOLEAN
USHORT KeProcessorRevision
Definition: krnlinit.c:21
#define KF_CR4
Definition: ketypes.h:145
LONGLONG TSCStart
Definition: ke.h:58
LONG NTSTATUS
Definition: precomp.h:26
#define MAXCHAR
Definition: umtypes.h:112
#define KPCR_TEB
Definition: asm.h:159
if(!(yy_init))
Definition: macro.lex.yy.c:717
#define NT_SUCCESS(StatCode)
Definition: apphelp.c:24
#define KPCR_PRCB
Definition: asm.h:66
char CCHAR
Definition: typedefs.h:50
uint64_t ULONGLONG
Definition: typedefs.h:65
VOID NTAPI KiRestoreFastSyscallReturnState(VOID)
Definition: cpu.c:1011
#define KF_MTRR
Definition: ketypes.h:149
PKNODE KeNodeBlock[1]
Definition: krnlinit.c:39
ULONG KiMinimumDpcRate
Definition: dpc.c:20
static VOID KiMarkPageAsReadOnly(PVOID Address)
Definition: kiinit.c:693
_In_ ULONG _In_ ULONG _In_ ULONG _Out_ PKIRQL _Out_ PKAFFINITY Affinity
Definition: halfuncs.h:170
ULONG KeI386CpuType
Definition: cpu.c:28
_In_opt_ PFILE_OBJECT _In_opt_ PETHREAD Thread
Definition: fltkernel.h:2653
VOID NTAPI KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Definition: kiinit.c:369
#define KERNEL_STACK_SIZE
#define KF_FXSR
Definition: ketypes.h:154
VOID KiInitializeMachineType(VOID)
Definition: kiinit.c:309
#define KF_3DNOW
Definition: ketypes.h:157
#define KGDT_R0_PCR
Definition: ketypes.h:79
#define ExAllocatePoolWithTag(hernya, size, tag)
Definition: env_spec_w32.h:350
#define PKTSS
Definition: ketypes.h:921
#define SharedUserData
#define KPCR_PROCESSOR_NUMBER
Definition: asm.h:178
unsigned char UCHAR
Definition: xmlstorage.h:181
ULONG KeI386CpuStep
Definition: cpu.c:29
VOID NTAPI KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
Definition: cpu.c:372
LONGLONG TSCDelta
Definition: ke.h:60
union _KGDTENTRY::@2324 HighWord
PVOID Ki386IopmSaveArea
Definition: v86vdm.c:23
VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack)
Definition: ke.h:796
ULONG_PTR NTAPI Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
Definition: cpu.c:1065
#define KF_NX_DISABLED
Definition: ketypes.h:166
#define PAGE_SIZE
Definition: env_spec_w32.h:49
#define PTE_BASE
Definition: mmx86.c:14
ULONG_PTR KiIdleSummary
Definition: thrdschd.c:25
KPROCESS Pcb
Definition: pstypes.h:1193
#define PASSIVE_LEVEL
Definition: env_spec_w32.h:693
#define KPCR_DR7
Definition: asm.h:184
VOID FASTCALL KiIdleLoop(VOID)
Definition: stubs.c:113
LONGLONG TSCEnd
Definition: ke.h:59
_In_opt_ PENTER_STATE_SYSTEM_HANDLER _In_opt_ PVOID _In_ LONG _In_opt_ LONG volatile * Number
Definition: ntpoapi.h:204
KAFFINITY KeActiveProcessors
Definition: krnlinit.c:23
CCHAR KeNumberProcessors
Definition: krnlinit.c:35
EPROCESS KiInitialProcess
Definition: krnlinit.c:45
Status
Definition: gdiplustypes.h:24
#define PROCESSOR_ARCHITECTURE_INTEL
Definition: ketypes.h:105
ULONG KiGetFeatureBits()
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
#define PCR_MINOR_VERSION
Definition: ke.h:277
ULONG KiMaximumDpcQueueDepth
Definition: dpc.c:19
#define HIGH_LEVEL
Definition: env_spec_w32.h:703
#define PF_RDTSC_INSTRUCTION_AVAILABLE
#define KF_XMMI64
Definition: ketypes.h:159
#define KPCR_DR6
Definition: asm.h:183
#define EXCEPTION_CHAIN_END
Definition: rtltypes.h:63
#define KPCR_NUMBER
Definition: asm.h:172
ULONG MXCsrMask
Definition: ketypes.h:434
BOOLEAN NTAPI KdInitSystem(ULONG Reserved, PLOADER_PARAMETER_BLOCK LoaderBlock)
Definition: kdinit.c:174
KPRIORITY NTAPI KeSetPriorityThread(IN PKTHREAD Thread, IN KPRIORITY Priority)
Definition: thrdobj.c:1327
unsigned short USHORT
Definition: pedump.c:61
#define RTL_NUMBER_OF(x)
Definition: RtlRegistry.c:12
VOID NTAPI KeInitializeThread(IN PKPROCESS Process, IN OUT PKTHREAD Thread, IN PKSYSTEM_ROUTINE SystemRoutine, IN PKSTART_ROUTINE StartRoutine, IN PVOID StartContext, IN PCONTEXT Context, IN PVOID Teb, IN PVOID KernelStack)
Definition: thrdobj.c:900
PKPRCB KiProcessorBlock[]
Definition: krnlinit.c:32
ETHREAD KiInitialThread
Definition: krnlinit.c:44
#define InitializeListHead(ListHead)
Definition: env_spec_w32.h:944
VOID KiSetProcessorType()
#define PRCB_BUILD_UNIPROCESSOR
Definition: ketypes.h:243
ULONG KSPIN_LOCK
Definition: env_spec_w32.h:72
VOID NTAPI KiInitSpinLocks(IN PKPRCB Prcb, IN CCHAR Number)
Definition: krnlinit.c:187
PPC_QUAL void __writefsdword(const unsigned long Offset, const unsigned long Data)
Definition: intrin_ppc.h:354
VOID NTAPI KiAmdK6InitializeMTRR(VOID)
Definition: mtrr.c:31
#define KPCR_SET_MEMBER
Definition: asm.h:171
#define DPRINT1
Definition: precomp.h:8
KDESCRIPTOR KiIdtDescriptor
Definition: except.c:51
ULONG64 Write
Definition: mmtypes.h:67
USHORT KeProcessorLevel
Definition: krnlinit.c:20
#define DBG_STATUS_CONTROL_C
Definition: kdtypes.h:39
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1870
VOID NTAPI Ki386EnableCurrentLargePage(IN ULONG_PTR StartAddress, IN ULONG Cr3)
ULONG MHz
Definition: ke.h:61
#define PF_XMMI64_INSTRUCTIONS_AVAILABLE
unsigned int ULONG
Definition: retypes.h:1
KSPIN_LOCK KiFreezeExecutionLock
Definition: kiinit.c:26
KSPIN_LOCK Ki486CompatibilityLock
Definition: kiinit.c:26
#define RtlZeroMemory(Destination, Length)
Definition: typedefs.h:261
#define ULONG_PTR
Definition: config.h:101
struct _KPCR * PKPCR
ULONGLONG BootCyclesEnd
Definition: kiinit.c:30
struct _ETHREAD * PETHREAD
Definition: nt_native.h:29
__analysis_noreturn NTSYSAPI VOID NTAPI DbgBreakPointWithStatus(_In_ ULONG Status)
KAFFINITY ProcessorMask
Definition: ketypes.h:776
#define KF_GLOBAL_PAGE
Definition: ketypes.h:147
PVOID NTAPI MmCreateKernelStack(BOOLEAN GuiStack, UCHAR Node)
ULONG_PTR NTAPI Ki386EnableTargetLargePage(IN ULONG_PTR Context)
Definition: patpge.c:70
#define KeGetCurrentThread
Definition: hal.h:44
#define KF_NX_ENABLED
Definition: ketypes.h:167
#define KPCR_SET_MEMBER_COPY
Definition: asm.h:158
#define INIT_SECTION
Definition: vfat.h:16
ULONG_PTR NTAPI Ki386EnableFxsr(IN ULONG_PTR Context)
Definition: cpu.c:1055
#define ALIGN_DOWN_POINTER_BY(ptr, align)
Definition: umtypes.h:82
signed int * PLONG
Definition: retypes.h:5
FXSAVE_FORMAT FxArea
Definition: ketypes.h:446
#define PF_3DNOW_INSTRUCTIONS_AVAILABLE
#define APC_LEVEL
Definition: env_spec_w32.h:695
VOID NTAPI KiInitializeKernel(IN PKPROCESS InitProcess, IN PKTHREAD InitThread, IN PVOID IdleStack, IN PKPRCB Prcb, IN CCHAR Number, IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Definition: kiinit.c:44
#define PCR_MAJOR_VERSION
Definition: ke.h:278
VOID NTAPI KeBugCheckEx(_In_ ULONG BugCheckCode, _In_ ULONG_PTR BugCheckParameter1, _In_ ULONG_PTR BugCheckParameter2, _In_ ULONG_PTR BugCheckParameter3, _In_ ULONG_PTR BugCheckParameter4)
Definition: rtlcompat.c:107
ULONG_PTR KernelStack
Definition: arc.h:496
ULONG_PTR NTAPI KeIpiGenericCall(IN PKIPI_BROADCAST_WORKER Function, IN ULONG_PTR Argument)
Definition: ipi.c:196
LIST_ENTRY KiProcessListHead
Definition: procobj.c:18
LONGLONG QuadPart
Definition: typedefs.h:112
#define PF_MMX_INSTRUCTIONS_AVAILABLE
VOID NTAPI KiI386PentiumLockErrataFixup(VOID)
Definition: cpu.c:1088
KIRQL WaitIrql
Definition: ketypes.h:1685
VOID FASTCALL RtlPrefetchMemoryNonTemporal(IN PVOID Source, IN SIZE_T Length)
Definition: misc.c:71