diff --git a/gemma/kv_cache.h b/gemma/kv_cache.h index 66f94f2..bad66fa 100644 --- a/gemma/kv_cache.h +++ b/gemma/kv_cache.h @@ -33,9 +33,7 @@ using KV_t = float; // A non-owning view of a KVCache. struct KVCachePtr { bool IsEmpty() const { return kv_cache.Rows() == 0; } - size_t SeqLen() const { - return kv_cache.Rows(); - } + size_t SeqLen() const; MatPtrT kv_cache; }; @@ -66,6 +64,10 @@ struct KVCache { KVCache(const Extents2D& kv_extents, const Allocator& allocator); }; +inline size_t KVCachePtr::SeqLen() const { + return kv_cache.Rows(); +} + // Convenience function to create views into KVCaches. std::vector ToKVCachePtrs(const hwy::Span& kv_caches);