item count for mapblock accessor

doc/config.md

@@ -57,6 +57,11 @@ The mapserver will generate a fresh `mapserver.json` if there is none at startup
 		"smartshop": true,
 		"fancyvend": true,
 		"atm": true
+	},
+	"mapblockaccessor": {
+		"expiretime": "10s",
+		"purgetime": "15s",
+		"maxitems": 10
 	}
 }
 ```
@@ -107,3 +112,6 @@ Faster system can use the default (10'000)
 
 #### enableprometheus
 Enables the [Prometheus](./prometheus.md) metrics endpoint
+
+#### mapblockaccessor.maxitems
+Number of mapblocks to keep in memory, dial this down if you have memory issues

server/mapblockaccessor/get.go

@@ -13,25 +13,24 @@ import (
 func (a *MapBlockAccessor) GetMapBlock(pos *coords.MapBlockCoords) (*mapblockparser.MapBlock, error) {
 	key := getKey(pos)
 
-	if a.c.ItemCount() > a.maxcount {
+	cacheBlocks.Set(float64(a.blockcache.ItemCount()))
+	if a.blockcache.ItemCount() > a.maxcount {
 		//flush cache
 		fields := logrus.Fields{
-			"cached items": a.c.ItemCount(),
+			"cached items": a.blockcache.ItemCount(),
 			"maxcount":     a.maxcount,
 		}
-		logrus.WithFields(fields).Warn("Flushing cache")
+		logrus.WithFields(fields).Debug("Flushing cache")
 
-		a.c.Flush()
+		a.blockcache.Flush()
 	}
 
-	cachedblock, found := a.c.Get(key)
+	cachedblock, found := a.blockcache.Get(key)
 	if found {
 		getCacheHitCount.Inc()
 		return cachedblock.(*mapblockparser.MapBlock), nil
 	}
 
-	getCacheMissCount.Inc()
-
 	timer := prometheus.NewTimer(dbGetDuration)
 	defer timer.ObserveDuration()
 
@@ -41,9 +40,14 @@ func (a *MapBlockAccessor) GetMapBlock(pos *coords.MapBlockCoords) (*mapblockpar
 	}
 
 	if block == nil {
+		//no mapblock here
+		cacheBlockCount.Inc()
+		a.blockcache.Set(key, nil, cache.DefaultExpiration)
 		return nil, nil
 	}
 
+	getCacheMissCount.Inc()
+
 	mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, pos)
 	if err != nil {
 		return nil, err
@@ -51,7 +55,8 @@ func (a *MapBlockAccessor) GetMapBlock(pos *coords.MapBlockCoords) (*mapblockpar
 
 	a.Eventbus.Emit(eventbus.MAPBLOCK_RENDERED, mapblock)
 
-	a.c.Set(key, mapblock, cache.DefaultExpiration)
+	cacheBlockCount.Inc()
+	a.blockcache.Set(key, mapblock, cache.DefaultExpiration)
 
 	return mapblock, nil
 }

server/mapblockaccessor/legacyblocks.go

@@ -61,7 +61,8 @@ func (a *MapBlockAccessor) FindNextLegacyBlocks(s settings.Settings, layers []*l
 
 		a.Eventbus.Emit(eventbus.MAPBLOCK_RENDERED, mapblock)
 
-		a.c.Set(key, mapblock, cache.DefaultExpiration)
+		a.blockcache.Set(key, mapblock, cache.DefaultExpiration)
+		cacheBlockCount.Inc()
 		mblist = append(mblist, mapblock)
 
 	}

server/mapblockaccessor/mapblockaccessor.go

@@ -12,10 +12,10 @@ import (
 )
 
 type MapBlockAccessor struct {
-	accessor db.DBAccessor
-	c        *cache.Cache
-	Eventbus *eventbus.Eventbus
-	maxcount int
+	accessor   db.DBAccessor
+	blockcache *cache.Cache
+	Eventbus   *eventbus.Eventbus
+	maxcount   int
 }
 
 func getKey(pos *coords.MapBlockCoords) string {
@@ -23,12 +23,12 @@ func getKey(pos *coords.MapBlockCoords) string {
 }
 
 func NewMapBlockAccessor(accessor db.DBAccessor, expiretime, purgetime time.Duration, maxcount int) *MapBlockAccessor {
-	c := cache.New(expiretime, purgetime)
+	blockcache := cache.New(expiretime, purgetime)
 
 	return &MapBlockAccessor{
-		accessor: accessor,
-		c:        c,
-		Eventbus: eventbus.New(),
-		maxcount: maxcount,
+		accessor:   accessor,
+		blockcache: blockcache,
+		Eventbus:   eventbus.New(),
+		maxcount:   maxcount,
 	}
 }

server/mapblockaccessor/mtime.go

@@ -76,7 +76,8 @@ func (a *MapBlockAccessor) FindMapBlocksByMtime(lastmtime int64, limit int, laye
 
 		a.Eventbus.Emit(eventbus.MAPBLOCK_RENDERED, mapblock)
 
-		a.c.Set(key, mapblock, cache.DefaultExpiration)
+		a.blockcache.Set(key, mapblock, cache.DefaultExpiration)
+		cacheBlockCount.Inc()
 		mblist = append(mblist, mapblock)
 
 	}

server/mapblockaccessor/prometheus.go

@@ -17,6 +17,18 @@ var (
 			Help: "Count of db cache miss",
 		},
 	)
+	cacheBlockCount = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "dbcache_block_count",
+			Help: "Count of db blocks inserted",
+		},
+	)
+	cacheBlocks = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "dbcache_blocks",
+			Help: "Block count currently in the cache",
+		},
+	)
 	dbGetDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "db_get_duration",
 		Help:    "Histogram for db mapblock get durations",
@@ -33,6 +45,9 @@ func init() {
 	prometheus.MustRegister(getCacheHitCount)
 	prometheus.MustRegister(getCacheMissCount)
 
+	prometheus.MustRegister(cacheBlockCount)
+	prometheus.MustRegister(cacheBlocks)
+
 	prometheus.MustRegister(dbGetDuration)
 	prometheus.MustRegister(dbGetMtimeDuration)
 }

server/mapblockaccessor/update.go

@@ -9,5 +9,6 @@ import (
 
 func (a *MapBlockAccessor) Update(pos *coords.MapBlockCoords, mb *mapblockparser.MapBlock) {
 	key := getKey(pos)
-	a.c.Set(key, mb, cache.DefaultExpiration)
+	cacheBlockCount.Inc()
+	a.blockcache.Set(key, mb, cache.DefaultExpiration)
 }