Mailing List Archive

Author: phk
Date: 2008-04-08 17:34:57 +0200 (Tue, 08 Apr 2008)
New Revision: 2629

Modified:
trunk/varnish-cache/bin/varnishd/storage_malloc.c
trunk/varnish-cache/include/stat_field.h
Log:
Polish up the malloc allocator to make it more useful for test and
debugging:

Implement an upper limit for allocated memory, specify as:
-smalloc,40M

Give the malloc allocator its own stats counters so we can see deeply
into what it does.



Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-08 15:33:30 UTC (rev 2628)
+++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-08 15:34:57 UTC (rev 2629)
@@ -35,14 +35,20 @@

#include <sys/types.h>

+#include <stdio.h>
#include <stdlib.h>
+#include <pthread.h>

#include "shmlog.h"
#include "cache.h"
#include "stevedore.h"

+static size_t sma_max = SIZE_T_MAX;
+static MTX sma_mtx;
+
struct sma {
struct storage s;
+ size_t sz;
};

static struct storage *
@@ -50,10 +56,24 @@
{
struct sma *sma;

- VSL_stats->sm_nreq++;
+ LOCK(&sma_mtx);
+ VSL_stats->sma_nreq++;
+ if (VSL_stats->sma_nbytes + size > sma_max)
+ size = 0;
+ else {
+ VSL_stats->sma_nobj++;
+ VSL_stats->sma_nbytes += size;
+ VSL_stats->sma_balloc += size;
+ }
+ UNLOCK(&sma_mtx);
+
+ if (size == 0)
+ return (NULL);
+
sma = calloc(sizeof *sma, 1);
if (sma == NULL)
return (NULL);
+ sma->sz = size;
sma->s.priv = sma;
sma->s.ptr = malloc(size);
XXXAN(sma->s.ptr);
@@ -62,8 +82,6 @@
sma->s.fd = -1;
sma->s.stevedore = st;
sma->s.magic = STORAGE_MAGIC;
- VSL_stats->sm_nobj++;
- VSL_stats->sm_balloc += sma->s.space;
return (&sma->s);
}

@@ -74,8 +92,12 @@

CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
sma = s->priv;
- VSL_stats->sm_nobj--;
- VSL_stats->sm_balloc -= sma->s.space;
+ assert(sma->sz == sma->s.space);
+ LOCK(&sma_mtx);
+ VSL_stats->sma_nobj--;
+ VSL_stats->sma_nbytes -= sma->sz;
+ VSL_stats->sma_bfree += sma->sz;
+ UNLOCK(&sma_mtx);
free(sma->s.ptr);
free(sma);
}
@@ -88,16 +110,52 @@

CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
sma = s->priv;
+ assert(sma->sz == sma->s.space);
if ((p = realloc(sma->s.ptr, size)) != NULL) {
- VSL_stats->sm_balloc -= sma->s.space;
+ LOCK(&sma_mtx);
+ VSL_stats->sma_nbytes -= (sma->sz - size);
+ VSL_stats->sma_bfree += sma->sz - size;
+ sma->sz = size;
+ UNLOCK(&sma_mtx);
sma->s.ptr = p;
sma->s.space = size;
- VSL_stats->sm_balloc += sma->s.space;
}
}

+static void
+sma_init(struct stevedore *parent, const char *spec)
+{
+ const char *e;
+ uintmax_t u;
+
+ (void)parent;
+ if (spec != NULL && *spec != '\0') {
+ e = str2bytes(spec, &u, 0);
+ if (e != NULL) {
+ fprintf(stderr,
+ "Error: (-smalloc) size \"%s\": %s\n", spec, e);
+ exit(2);
+ }
+ if ((u != (uintmax_t)(size_t)u)) {
+ fprintf(stderr,
+ "Error: (-smalloc) size \"%s\": too big\n", spec);
+ exit(2);
+ }
+ sma_max = u;
+ }
+}
+
+static void
+sma_open(const struct stevedore *st)
+{
+ (void)st;
+ AZ(pthread_mutex_init(&sma_mtx, NULL));
+}
+
struct stevedore sma_stevedore = {
.name = "malloc",
+ .init = sma_init,
+ .open = sma_open,
.alloc = sma_alloc,
.free = sma_free,
.trim = sma_trim,

Modified: trunk/varnish-cache/include/stat_field.h
===================================================================
--- trunk/varnish-cache/include/stat_field.h 2008-04-08 15:33:30 UTC (rev 2628)
+++ trunk/varnish-cache/include/stat_field.h 2008-04-08 15:34:57 UTC (rev 2629)
@@ -97,4 +97,10 @@
MAC_STAT(sm_balloc, uint64_t, 'i', "bytes allocated")
MAC_STAT(sm_bfree, uint64_t, 'i', "bytes free")

+MAC_STAT(sma_nreq, uint64_t, 'a', "SMA allocator requests")
+MAC_STAT(sma_nobj, uint64_t, 'i', "SMA outstanding allocations")
+MAC_STAT(sma_nbytes, uint64_t, 'i', "SMA outstanding bytes")
+MAC_STAT(sma_balloc, uint64_t, 'i', "SMA bytes allocated")
+MAC_STAT(sma_bfree, uint64_t, 'i', "SMA bytes free")
+
MAC_STAT(backend_req, uint64_t, 'a', "Backend requests made")