20 #include "pedigree/kernel/utilities/Cache.h" 21 #include "pedigree/kernel/LockGuard.h" 22 #include "pedigree/kernel/Log.h" 23 #include "pedigree/kernel/machine/Machine.h" 24 #include "pedigree/kernel/machine/Timer.h" 25 #include "pedigree/kernel/process/MemoryPressureManager.h" 26 #include "pedigree/kernel/processor/PhysicalMemoryManager.h" 27 #include "pedigree/kernel/processor/VirtualAddressSpace.h" 28 #include "pedigree/kernel/utilities/Iterator.h" 29 #include "pedigree/kernel/utilities/assert.h" 30 #include "pedigree/kernel/utilities/utility.h" 32 #ifndef STANDALONE_CACHE 33 #include "pedigree/kernel/process/Scheduler.h" 34 #include "pedigree/kernel/process/Thread.h" 35 #include "pedigree/kernel/processor/Processor.h" 36 #include "pedigree/kernel/processor/ProcessorInformation.h" 39 #include "pedigree/kernel/utilities/smhasher/MurmurHash3.h" 49 static bool g_AllocatorInited =
false;
54 static int trimTrampoline(
void *p)
56 CacheManager::instance().trimThread();
61 CacheManager::CacheManager()
70 CacheManager::~CacheManager()
74 m_pTrimThread->join();
80 #ifndef STANDALONE_CACHE 84 t->registerHandler(
this);
95 m_pTrimThread =
new Thread(pParent, trimTrampoline, 0);
99 void CacheManager::registerCache(
Cache *pCache)
101 m_Caches.pushBack(pCache);
104 void CacheManager::unregisterCache(
Cache *pCache)
119 size_t totalEvicted = 0;
121 (it != m_Caches.end()) && count; ++it)
123 size_t evicted = (*it)->trim(count);
124 totalEvicted += evicted;
128 return totalEvicted != 0;
136 (*it)->timer(delta, state);
141 uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, uint64_t p5,
142 uint64_t p6, uint64_t p7, uint64_t p8)
144 Cache *pCache =
reinterpret_cast<Cache *
>(p1);
149 bool bCacheFound =
false;
162 ERROR(
"CacheManager::executeRequest for an unregistered cache!");
170 void CacheManager::trimThread()
176 size_t lowMark = MemoryPressureManager::getLowWatermark();
181 NOTICE_NOLOCK(
"trimThread: free page count nears high watermark, " 182 "automatically trimming");
185 size_t trimCount = (lowMark - currFree) + 1;
194 Cache::Cache(
size_t pageConstraints)
195 : m_Pages(), m_PageFilter(0xe80000, 11), m_pLruHead(0), m_pLruTail(0),
196 m_Lock(
false), m_Callback(0), m_Nanoseconds(0),
197 m_PageConstraints(pageConstraints)
199 if (!g_AllocatorInited)
201 #ifdef STANDALONE_CACHE 204 discover_range(start, end);
211 m_Allocator.free(start, end - start);
212 g_AllocatorInited =
true;
220 CacheManager::instance().registerCache(
this);
227 it != m_Pages.end(); it++)
232 CacheManager::instance().unregisterCache(
this);
240 if (!m_PageFilter.contains(key))
265 bool triedLookup =
false;
266 if (m_PageFilter.contains(key))
268 pPage = m_Pages.lookup(key);
273 *alreadyExisted =
true;
283 *alreadyExisted =
false;
288 if ((!triedLookup) && m_Pages.lookup(key))
290 FATAL(
"Cache: bloom filter lied!");
293 m_AllocatorLock.acquire();
294 uintptr_t location = 0;
295 bool succeeded = m_Allocator.allocate(4096, location);
296 m_AllocatorLock.release();
301 "Cache: out of address space [have " << m_Pages.count()
311 FATAL(
"Map failed in Cache::insert())");
321 pPage->status = CachePage::Editing;
322 m_Pages.insert(key, pPage);
323 m_PageFilter.add(key);
335 WARNING(
"Cache::insert called with a size that isn't page-aligned");
339 size_t nPages = size / 4096;
344 if (m_PageFilter.contains(key))
346 pPage = m_Pages.lookup(key);
351 *alreadyExisted =
true;
359 *alreadyExisted =
false;
363 m_AllocatorLock.acquire();
365 bool succeeded = m_Allocator.allocate(size, location);
366 m_AllocatorLock.release();
370 ERROR(
"Cache: can't allocate " <<
Dec << size <<
Hex <<
" bytes.");
374 uintptr_t returnLocation = location;
375 bool bOverlap =
false;
376 for (
size_t page = 0; page < nPages; page++)
378 pPage = m_Pages.lookup(key + (page * 4096));
390 FATAL(
"Map failed in Cache::insert())");
394 pPage->
key = key + (page * 4096);
402 pPage->status = CachePage::Editing;
404 m_Pages.insert(key + (page * 4096), pPage);
405 m_PageFilter.add(key + (page * 4096));
414 return returnLocation;
419 #ifdef STANDALONE_CACHE 423 physical_uintptr_t phys =
426 phys, reinterpret_cast<void *>(virt),
436 for (
size_t i = 0; i < length; i += 0x1000)
438 if (!m_PageFilter.contains(key + (i * 0x1000)))
444 CachePage *pPage = m_Pages.lookup(key + (i * 0x1000));
457 return evict(key,
true,
true,
true);
466 it != m_Pages.end(); ++it)
471 evict(it.key(),
false,
true,
false);
477 bool Cache::evict(uintptr_t key,
bool bLock,
bool bPhysicalLock,
bool bRemove)
485 if (!m_PageFilter.contains(key))
487 pPage = m_Pages.lookup(key);
492 "Cache::evict didn't evict " << key
493 <<
" as it didn't actually exist");
506 if ((m_Callback && pPage->
refcnt <= 1) ||
507 ((!m_Callback) && (!pPage->
refcnt)))
510 if (!verifyChecksum(pPage))
513 CacheConstants::WriteBack, key, pPage->
location,
517 #ifndef STANDALONE_CACHE 520 void *loc =
reinterpret_cast<void *
>(pPage->
location);
522 physical_uintptr_t phys;
537 CacheConstants::Eviction, key, pPage->
location, m_CallbackMeta);
539 #ifndef STANDALONE_CACHE 546 m_Allocator.free(pPage->
location, 4096);
561 if (!m_PageFilter.contains(key))
582 if (!m_PageFilter.contains(key))
601 1, reinterpret_cast<uint64_t>(
this), CacheConstants::PleaseEvict,
617 while ((nPages < count) && ((n = lruEvict(
true)) > 0))
632 if (!m_PageFilter.contains(key))
643 uintptr_t location = pPage->
location;
649 1, reinterpret_cast<uint64_t>(
this), CacheConstants::WriteBack, key,
654 uint64_t result = CacheManager::instance().
addRequest(
655 1, reinterpret_cast<uint64_t>(
this), CacheConstants::WriteBack, key,
659 WARNING(
"Cache: writeback failed in sync");
668 if (!m_PageFilter.contains(key))
679 calculateChecksum(pPage);
684 m_Nanoseconds += delta;
685 if (
LIKELY(m_Nanoseconds < (CACHE_WRITEBACK_PERIOD * 1000000ULL)))
689 else if (
UNLIKELY(m_bInCritical == 1))
699 it != m_Pages.end(); ++it)
702 if (page->status == CachePage::Editing)
707 else if (page->status == CachePage::EditTransition)
711 page->status = CachePage::ChecksumStable;
714 else if (page->status == CachePage::ChecksumChanging)
717 if (verifyChecksum(page,
true))
720 page->status = CachePage::ChecksumStable;
728 else if (page->status == CachePage::ChecksumStable)
731 if (!verifyChecksum(page,
true))
734 page->status = CachePage::ChecksumChanging;
742 ERROR(
"Unknown page status!");
750 NOTICE(
"** writeback @" <<
Hex << it.key());
752 1, reinterpret_cast<uint64_t>(
this), CacheConstants::WriteBack,
761 m_Callback = newCallback;
762 m_CallbackMeta = meta;
766 uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, uint64_t p5,
767 uint64_t p6, uint64_t p7, uint64_t p8)
773 if (static_cast<CacheConstants::CallbackCause>(p2) ==
774 CacheConstants::PleaseEvict)
776 evict(p3,
false,
true,
true);
784 NOTICE(
"Cache: writeback for off=" << p3 <<
" @" << p3 <<
"!");
787 static_cast<CacheConstants::CallbackCause>(p2), p3, p4, m_CallbackMeta);
790 "Cache: writeback for off=" << p3 <<
" @" << p3 <<
" complete!");
801 #ifdef STANDALONE_CACHE 804 if (!(m_pLruHead && m_pLruTail))
809 MemoryPressureManager::getLowWatermark()))
813 if (evict(toEvict->
key,
false,
true,
true))
818 promotePage(toEvict);
829 pPage->
pNext = m_pLruHead;
831 m_pLruHead->pPrev = pPage;
834 m_pLruTail = m_pLruHead;
848 pPage->
pNext->pPrev = pPage->pPrev;
849 if (pPage == m_pLruTail)
850 m_pLruTail = pPage->pPrev;
851 if (pPage == m_pLruHead)
852 m_pLruHead = pPage->
pNext;
857 void *buffer =
reinterpret_cast<void *
>(pPage->
location);
858 checksum(buffer, 4096, pPage->
checksum);
863 void *buffer =
reinterpret_cast<void *
>(pPage->
location);
865 uint64_t new_checksum[2];
866 checksum(buffer, 4096, new_checksum);
872 pPage->
checksum[0] = new_checksum[0];
873 pPage->
checksum[1] = new_checksum[1];
881 MurmurHash3_x64_128(data, len, 0, out);
891 "Cache::markEditing called with a length that isn't page-aligned");
900 size_t nPages = length / 4096;
902 for (
size_t page = 0; page < nPages; page++)
904 if (!m_PageFilter.contains(key + (page * 4096)))
909 CachePage *pPage = m_Pages.lookup(key + (page * 4096));
915 pPage->status = CachePage::Editing;
926 "Cache::markEditing called with a length that isn't page-aligned");
935 size_t nPages = length / 4096;
937 for (
size_t page = 0; page < nPages; page++)
939 if (!m_PageFilter.contains(key + (page * 4096)))
944 CachePage *pPage = m_Pages.lookup(key + (page * 4096));
950 pPage->status = CachePage::EditTransition;
955 calculateChecksum(pPage);
959 CachePageGuard::CachePageGuard(
Cache &cache, uintptr_t location)
960 : m_Cache(cache), m_Location(location)
964 CachePageGuard::~CachePageGuard()
966 m_Cache.release(m_Location);
971 return checksum[0] == other[0] && checksum[1] == other[1];
976 return checksum[0] == 0 && checksum[1] == 0;
void triggerChecksum(uintptr_t key)
virtual void unmap(void *virtualAddress)=0
uintptr_t key
Key for this page.
static PhysicalMemoryManager & instance()
uintptr_t insert(uintptr_t key, bool *alreadyExisted=nullptr)
bool map(uintptr_t virt) const
virtual void getMapping(void *virtualAddress, physical_uintptr_t &physicalAddress, size_t &flags)=0
uint64_t checksum[2]
Checksum of the page's contents (for dirty detection).
static Spinlock m_AllocatorLock
uint64_t addAsyncRequest(size_t priority, uint64_t p1=0, uint64_t p2=0, uint64_t p3=0, uint64_t p4=0, uint64_t p5=0, uint64_t p6=0, uint64_t p7=0, uint64_t p8=0)
void markNoLongerEditing(uintptr_t key, size_t length=0)
uintptr_t location
The location of this page in memory.
virtual Timer * getTimer()=0
size_t trim(size_t count=1)
virtual physical_uintptr_t allocatePage(size_t pageConstraints=0)=0
bool exists(uintptr_t key, size_t length)
void sync(uintptr_t key, bool async)
static EXPORTED_PUBLIC VirtualAddressSpace & getKernelAddressSpace()
bool acquire(bool recurse=false, bool safe=true)
bool verifyChecksum(CachePage *pPage, bool replace=false)
static ProcessorInformation & information()
bool trimAll(size_t count=1)
virtual void timer(uint64_t delta, InterruptState &state)
size_t lruEvict(bool force=false)
void markEditing(uintptr_t key, size_t length=0)
bool checkChecksum(uint64_t other[2]) const
Check the checksum against another.
void unlinkPage(CachePage *pPage)
static const size_t Write
virtual void timer(uint64_t delta, InterruptState &state)
virtual uint64_t executeRequest(uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, uint64_t p5, uint64_t p6, uint64_t p7, uint64_t p8)
uintptr_t lookup(uintptr_t key)
void linkPage(CachePage *pPage)
static const size_t KernelMode
bool checkZeroChecksum() const
Check for an unset checksum.
void calculateChecksum(CachePage *pPage)
void promotePage(CachePage *pPage)
virtual uintptr_t getKernelCacheEnd() const =0
static MemoryAllocator m_Allocator
static Scheduler & instance()
virtual size_t freePageCount() const
bool evict(uintptr_t key)
void release(uintptr_t key)
CachePage * pNext
Linked list components for LRU.
virtual uintptr_t getKernelCacheStart() const =0
virtual void initialise()
MUST_USE_RESULT uint64_t addRequest(size_t priority, uint64_t p1=0, uint64_t p2=0, uint64_t p3=0, uint64_t p4=0, uint64_t p5=0, uint64_t p6=0, uint64_t p7=0, uint64_t p8=0)
virtual void freePage(physical_uintptr_t page)=0
An iterator applicable for many data structures.
void setCallback(writeback_t newCallback, void *meta)
virtual uint64_t executeRequest(uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4, uint64_t p5, uint64_t p6, uint64_t p7, uint64_t p8)
void(* writeback_t)(CacheConstants::CallbackCause cause, uintptr_t loc, uintptr_t page, void *meta)
void checksum(const void *data, size_t len, uint64_t out[2])