377 iLowerAllocator->Alloc(0,aNumInitPages); |
377 iLowerAllocator->Alloc(0,aNumInitPages); |
378 iLowerWaterMark = aNumInitPages-1; |
378 iLowerWaterMark = aNumInitPages-1; |
379 |
379 |
380 iUpperAllocator = TBitMapAllocator::New(KMaxPageTablePages,ETrue); |
380 iUpperAllocator = TBitMapAllocator::New(KMaxPageTablePages,ETrue); |
381 __NK_ASSERT_ALWAYS(iUpperAllocator); |
381 __NK_ASSERT_ALWAYS(iUpperAllocator); |
382 iUpperWaterMark = KMaxPageTablePages; |
382 |
|
383 __ASSERT_COMPILE(KMaxPageTablePages > (TUint)KMinUnpinnedPagedPtPages); // Unlikely to be untrue. |
|
384 iUpperWaterMark = KMaxPageTablePages - KMinUnpinnedPagedPtPages; |
|
385 iPinnedPageTablePages = 0; // OK to clear this without MmuLock as only one thread running so far. |
|
386 } |
|
387 |
|
388 |
|
389 static TUint32 RandomSeed = 33333; |
|
390 |
|
391 TUint PageTableAllocator::TPtPageAllocator::RandomPagedPtPage() |
|
392 { |
|
393 __NK_ASSERT_DEBUG(PageTablesLockIsHeld()); |
|
394 |
|
395 // Pick an allocated page at random, from iUpperWaterMark - KMaxPageTablePages. |
|
396 RandomSeed = RandomSeed * 69069 + 1; // next 'random' number |
|
397 TUint allocRange = KMaxPageTablePages - iUpperWaterMark - 1; |
|
398 TUint bit = (TUint64(RandomSeed) * TUint64(allocRange)) >> 32; |
|
399 |
|
400 // All page table pages should be allocated or we shouldn't be stealing one at random. |
|
401 __NK_ASSERT_DEBUG(iUpperAllocator->NotFree(bit, 1)); |
|
402 |
|
403 return KMaxPageTablePages - 1 - bit; |
383 } |
404 } |
384 |
405 |
385 |
406 |
386 TInt PageTableAllocator::TPtPageAllocator::Alloc(TBool aDemandPaged) |
407 TInt PageTableAllocator::TPtPageAllocator::Alloc(TBool aDemandPaged) |
387 { |
408 { |
388 __NK_ASSERT_DEBUG(PageTablesLockIsHeld()); |
409 __NK_ASSERT_DEBUG(PageTablesLockIsHeld()); |
389 TUint pageIndex; |
410 TUint pageIndex; |
390 if(aDemandPaged) |
411 if(aDemandPaged) |
391 { |
412 { |
392 TInt bit = iUpperAllocator->Alloc(); |
413 TInt bit = iUpperAllocator->Alloc(); |
393 if(bit<0) |
414 // There are always unpaged page tables so iUpperAllocator will always have |
394 return bit; |
415 // at least one free bit. |
395 pageIndex = KMaxPageTablePages-1-bit; |
416 __NK_ASSERT_DEBUG(bit >= 0); |
396 if(pageIndex<iUpperWaterMark) |
417 |
|
418 pageIndex = KMaxPageTablePages - 1 - bit; |
|
419 |
|
420 if(pageIndex < iUpperWaterMark) |
397 { |
421 { |
398 // new upper watermark... |
422 // new upper watermark... |
399 if((pageIndex&~(KPageTableGroupSize-1))<=iLowerWaterMark) |
423 if((pageIndex & ~(KPageTableGroupSize - 1)) <= iLowerWaterMark) |
400 { |
424 { |
401 // clashes with other bitmap allocator, so fail.. |
425 // clashes with other bitmap allocator, so fail.. |
402 iUpperAllocator->Free(bit); |
426 iUpperAllocator->Free(bit); |
|
427 TRACE(("TPtPageAllocator::Alloc too low iUpperWaterMark %d ",iUpperWaterMark)); |
403 return -1; |
428 return -1; |
404 } |
429 } |
|
430 // Hold mmu lock so iUpperWaterMark isn't read by pinning before we've updated it. |
|
431 MmuLock::Lock(); |
405 iUpperWaterMark = pageIndex; |
432 iUpperWaterMark = pageIndex; |
|
433 MmuLock::Unlock(); |
406 TRACE(("TPtPageAllocator::Alloc new iUpperWaterMark=%d",pageIndex)); |
434 TRACE(("TPtPageAllocator::Alloc new iUpperWaterMark=%d",pageIndex)); |
407 } |
435 } |
408 } |
436 } |
409 else |
437 else |
410 { |
438 { |
411 TInt bit = iLowerAllocator->Alloc(); |
439 TInt bit = iLowerAllocator->Alloc(); |
412 if(bit<0) |
440 if(bit < 0) |
413 return bit; |
441 return bit; |
414 pageIndex = bit; |
442 pageIndex = bit; |
415 if(pageIndex>iLowerWaterMark) |
443 if(pageIndex > iLowerWaterMark) |
416 { |
444 {// iLowerAllocator->Alloc() should only pick the next bit after iLowerWaterMark. |
417 // new upper watermark... |
445 __NK_ASSERT_DEBUG(pageIndex == iLowerWaterMark + 1); |
418 if(pageIndex>=(iUpperWaterMark&~(KPageTableGroupSize-1))) |
446 MmuLock::Lock(); |
|
447 // new lower watermark... |
|
448 if( pageIndex >= (iUpperWaterMark & ~(KPageTableGroupSize - 1)) || |
|
449 AtPinnedPagedPtsLimit(iUpperWaterMark, pageIndex, iPinnedPageTablePages)) |
419 { |
450 { |
420 // clashes with other bitmap allocator, so fail.. |
451 // clashes with other bitmap allocator or it would reduce the amount |
|
452 // of available unpinned paged page tables too far, so fail.. |
|
453 MmuLock::Unlock(); |
421 iLowerAllocator->Free(bit); |
454 iLowerAllocator->Free(bit); |
|
455 TRACE(("TPtPageAllocator::Alloc iLowerWaterMark=%d",iLowerWaterMark)); |
422 return -1; |
456 return -1; |
423 } |
457 } |
424 iLowerWaterMark = pageIndex; |
458 iLowerWaterMark = pageIndex; |
425 TRACE(("TPtPageAllocator::Alloc new iLowerWaterMark=%d",pageIndex)); |
459 MmuLock::Unlock(); |
|
460 TRACE(("TPtPageAllocator::Alloc new iLowerWaterMark=%d", iLowerWaterMark)); |
426 } |
461 } |
427 } |
462 } |
428 return pageIndex; |
463 return pageIndex; |
429 } |
464 } |
430 |
465 |
459 |
494 |
460 TBool PageTableAllocator::AllocReserve(TSubAllocator& aSubAllocator) |
495 TBool PageTableAllocator::AllocReserve(TSubAllocator& aSubAllocator) |
461 { |
496 { |
462 __NK_ASSERT_DEBUG(LockIsHeld()); |
497 __NK_ASSERT_DEBUG(LockIsHeld()); |
463 |
498 |
|
499 TBool demandPaged = aSubAllocator.iDemandPaged; |
|
500 |
464 // allocate page... |
501 // allocate page... |
465 TInt ptPageIndex = iPtPageAllocator.Alloc(aSubAllocator.iDemandPaged); |
502 TInt ptPageIndex = iPtPageAllocator.Alloc(demandPaged); |
466 if(ptPageIndex<0) |
503 if (ptPageIndex < 0) |
467 return false; |
504 { |
|
505 if (demandPaged) |
|
506 { |
|
507 TInt r; |
|
508 do |
|
509 { |
|
510 // Can't fail to find a demand paged page table, otherwise a page fault |
|
511 // could fail with KErrNoMemory. Therefore, keep attempting to steal a |
|
512 // demand paged page table page until successful. |
|
513 TUint index = iPtPageAllocator.RandomPagedPtPage(); |
|
514 MmuLock::Lock(); |
|
515 TLinAddr pageTableLin = KPageTableBase + (index << (KPtClusterShift + KPageTableShift)); |
|
516 TPhysAddr ptPhysAddr = Mmu::LinearToPhysical(pageTableLin); |
|
517 // Page tables must be allocated otherwise we shouldn't be stealing the page. |
|
518 __NK_ASSERT_DEBUG(ptPhysAddr != KPhysAddrInvalid); |
|
519 SPageInfo* ptSPageInfo = SPageInfo::FromPhysAddr(ptPhysAddr); |
|
520 r = StealPage(ptSPageInfo); |
|
521 MmuLock::Unlock(); |
|
522 } |
|
523 while(r != KErrCompletion); |
|
524 // Retry the allocation now that we've stolen a page table page. |
|
525 ptPageIndex = iPtPageAllocator.Alloc(demandPaged); |
|
526 __NK_ASSERT_DEBUG(ptPageIndex >= 0); |
|
527 } |
|
528 else |
|
529 { |
|
530 return EFalse; |
|
531 } |
|
532 } |
468 |
533 |
469 // commit memory for page... |
534 // commit memory for page... |
470 __NK_ASSERT_DEBUG(iPageTableMemory); // check we've initialised iPageTableMemory |
535 __NK_ASSERT_DEBUG(iPageTableMemory); // check we've initialised iPageTableMemory |
471 TInt r = ThePageTableMemoryManager.Alloc(iPageTableMemory,ptPageIndex,aSubAllocator.iDemandPaged); |
536 TInt r = ThePageTableMemoryManager.Alloc(iPageTableMemory,ptPageIndex,aSubAllocator.iDemandPaged); |
472 if(r==KErrNoMemory) |
537 if(r==KErrNoMemory) |
1105 TUint aBlockZoneId, TBool aBlockRest) |
1170 TUint aBlockZoneId, TBool aBlockRest) |
1106 { |
1171 { |
1107 // We don't move page table or page table info pages, however, if this page |
1172 // We don't move page table or page table info pages, however, if this page |
1108 // is demand paged then we may be able to discard it. |
1173 // is demand paged then we may be able to discard it. |
1109 MmuLock::Lock(); |
1174 MmuLock::Lock(); |
1110 if (!(iPtPageAllocator.IsDemandPaged(aOldPageInfo))) |
1175 if (aOldPageInfo->Owner() == iPageTableInfoMemory) |
1111 { |
1176 { |
1112 MmuLock::Unlock(); |
1177 if (!(iPtPageAllocator.IsDemandPagedPtInfo(aOldPageInfo))) |
1113 return KErrNotSupported; |
1178 { |
|
1179 MmuLock::Unlock(); |
|
1180 return KErrNotSupported; |
|
1181 } |
|
1182 } |
|
1183 else |
|
1184 { |
|
1185 __NK_ASSERT_DEBUG(aOldPageInfo->Owner() == iPageTableMemory); |
|
1186 if (!(iPtPageAllocator.IsDemandPagedPt(aOldPageInfo))) |
|
1187 { |
|
1188 MmuLock::Unlock(); |
|
1189 return KErrNotSupported; |
|
1190 } |
1114 } |
1191 } |
1115 if (aOldPageInfo->PagedState() == SPageInfo::EPagedPinned) |
1192 if (aOldPageInfo->PagedState() == SPageInfo::EPagedPinned) |
1116 {// The page is pinned so don't attempt to discard it as pinned pages |
1193 {// The page is pinned so don't attempt to discard it as pinned pages |
1117 // can't be discarded. Also, the pager will invoke this method again. |
1194 // can't be discarded. Also, the pager will invoke this method again. |
1118 MmuLock::Unlock(); |
1195 MmuLock::Unlock(); |
1123 // PageTableAllocator::StealPage() will be invoked on this page. |
1200 // PageTableAllocator::StealPage() will be invoked on this page. |
1124 return ThePager.DiscardPage(aOldPageInfo, aBlockZoneId, aBlockRest); |
1201 return ThePager.DiscardPage(aOldPageInfo, aBlockZoneId, aBlockRest); |
1125 } |
1202 } |
1126 |
1203 |
1127 |
1204 |
1128 void PageTableAllocator::PinPageTable(TPte* aPageTable, TPinArgs& aPinArgs) |
1205 TInt PageTableAllocator::PinPageTable(TPte* aPageTable, TPinArgs& aPinArgs) |
1129 { |
1206 { |
1130 __NK_ASSERT_DEBUG(MmuLock::IsHeld()); |
1207 __NK_ASSERT_DEBUG(MmuLock::IsHeld()); |
1131 __NK_ASSERT_DEBUG(SPageTableInfo::FromPtPtr(aPageTable)->IsDemandPaged()); |
1208 __NK_ASSERT_DEBUG(SPageTableInfo::FromPtPtr(aPageTable)->IsDemandPaged()); |
1132 __NK_ASSERT_DEBUG(!SPageTableInfo::FromPtPtr(aPageTable)->IsUnused()); |
1209 __NK_ASSERT_DEBUG(!SPageTableInfo::FromPtPtr(aPageTable)->IsUnused()); |
1133 __NK_ASSERT_DEBUG(aPinArgs.HaveSufficientPages(KNumPagesToPinOnePageTable)); |
1210 __NK_ASSERT_DEBUG(aPinArgs.HaveSufficientPages(KNumPagesToPinOnePageTable)); |
1134 |
1211 |
1135 // pin page with page table in... |
1212 // pin page with page table in... |
1136 TPhysAddr pagePhys = Mmu::PageTablePhysAddr(aPageTable); |
1213 TPhysAddr pagePhys = Mmu::PageTablePhysAddr(aPageTable); |
1137 SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys); |
1214 SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys); |
|
1215 if (!pi->PinCount()) |
|
1216 {// Page is being pinned having previously been unpinned. |
|
1217 TInt r = iPtPageAllocator.PtPagePinCountInc(); |
|
1218 if (r != KErrNone) |
|
1219 return r; |
|
1220 } |
1138 ThePager.Pin(pi,aPinArgs); |
1221 ThePager.Pin(pi,aPinArgs); |
1139 |
1222 |
1140 // pin page with page table info in... |
1223 // pin page with page table info in... |
1141 SPageTableInfo* pti = SPageTableInfo::FromPtPtr(aPageTable); |
1224 SPageTableInfo* pti = SPageTableInfo::FromPtPtr(aPageTable); |
1142 pagePhys = Mmu::UncheckedLinearToPhysical((TLinAddr)pti,KKernelOsAsid); |
1225 pagePhys = Mmu::UncheckedLinearToPhysical((TLinAddr)pti,KKernelOsAsid); |
1143 pi = SPageInfo::FromPhysAddr(pagePhys); |
1226 pi = SPageInfo::FromPhysAddr(pagePhys); |
1144 ThePager.Pin(pi,aPinArgs); |
1227 ThePager.Pin(pi,aPinArgs); |
|
1228 return KErrNone; |
1145 } |
1229 } |
1146 |
1230 |
1147 |
1231 |
1148 void PageTableAllocator::UnpinPageTable(TPte* aPageTable, TPinArgs& aPinArgs) |
1232 void PageTableAllocator::UnpinPageTable(TPte* aPageTable, TPinArgs& aPinArgs) |
1149 { |
1233 { |