Mailing List Archive

Author: phk
Date: 2007-05-09 16:28:50 +0200 (Wed, 09 May 2007)
New Revision: 1398

Modified:
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_center.c
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/cache_http.c
trunk/varnish-cache/bin/varnishd/cache_vrt.c
trunk/varnish-cache/bin/varnishd/hash_classic.c
trunk/varnish-cache/bin/varnishd/hash_simple_list.c
trunk/varnish-cache/bin/varnishd/mgt_vcc.c
Log:
Implement runtime part of VCL controlled hashing.

The vcl_hash() is now used to control which fields go
into the hash algorithm, and the default is stil,
as previously, the URL + Host: header.

But now it is controlled by the vcl code, with the
default vcl_hash() being:

sub vcl_hash {
req.hash += req.url;
req.hash += req.http.host;
hash;
}

Once I get a bit further, this will be changed to

sub vcl_hash {
req.hash += req.url;
if (req.http.host) {
req.hash += req.http.host;
} else {
req.hash += server.ip;
}
hash;
}

So that we correctly hash HTTP requests without Host:
headers, that go to a machine with multiple IP numbers.

If you want to add fields to the hash, just write
a vcl_hash that does not end in "hash;":

sub vcl_hash {
req.hash += req.http.cookie;
}

If you want to override the default vcl_hash, just
say so:

sub vcl_hash {
req.hash += req.url;
hash; // do not continue into default vcl_hash
}




Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2007-05-09 14:28:50 UTC (rev 1398)
@@ -291,6 +291,9 @@

struct workreq workreq;
struct acct acct;
+
+ char *hash_b; /* Start of hash string */
+ char *hash_e; /* End of hash string */
};

struct backend {

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -437,9 +437,18 @@
{
struct object *o;

- VCL_hash_method(sp); /* XXX: no-op for now */
+ assert(sp->http->f > sp->http->s);
+ assert(sp->http->f >= sp->http->t);
+ if (sp->obj == NULL) {
+ sp->hash_b = sp->http->f;
+ sp->hash_e = sp->hash_b;
+ VCL_hash_method(sp); /* XXX: no-op for now */

+ /* XXX check error */
+ }
+
o = HSH_Lookup(sp);
+
if (o == NULL) {
/*
* We hit a busy object, disembark worker thread and expect
@@ -451,6 +460,14 @@
return (1);
}

+ xxxassert (sp->hash_e == sp->http->f);
+ if (sp->hash_e == sp->http->f) {
+ /* Nobody alloc'ed after us, free again */
+ sp->http->f = sp->hash_b;
+ }
+
+ sp->hash_b = sp->hash_e = NULL;
+
sp->obj = o;

/* If we inserted a new object it's a miss */

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -115,7 +115,6 @@
struct http *h;
struct objhead *oh;
struct object *o;
- char *url, *host;

CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -125,9 +124,6 @@
h = sp->http;

HSH_Prealloc(sp);
- url = h->hd[HTTP_HDR_URL].b;
- if (!http_GetHdr(h, H_Host, &host))
- host = url;
if (sp->obj != NULL) {
CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
o = sp->obj;
@@ -136,7 +132,9 @@
LOCK(&oh->mtx);
goto were_back;
}
- oh = hash->lookup(url, host, w->nobjhead);
+VSLR(SLT_Debug, sp->fd, sp->hash_b, sp->hash_e);
+
+ oh = hash->lookup(sp->hash_b, sp->hash_e, w->nobjhead);
CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
if (oh == w->nobjhead)
w->nobjhead = NULL;
@@ -157,7 +155,7 @@
/* Object banned but not reaped yet */
} else if (o->ttl <= sp->t_req.tv_sec) {
/* Object expired */
- } else if (BAN_CheckObject(o, url)) {
+ } else if (BAN_CheckObject(o, h->hd[HTTP_HDR_URL].b)) {
o->ttl = 0;
VSL(SLT_ExpBan, 0, "%u was banned", o->xid);
EXP_TTLchange(o);

Modified: trunk/varnish-cache/bin/varnishd/cache_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_http.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/cache_http.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -540,6 +540,7 @@
hp->t = p;
assert(hp->t > hp->s);
assert(hp->t <= hp->v);
+ hp->f = hp->v;
return (1);
}


Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -273,3 +273,21 @@

return ((struct sockaddr*)sp->mysockaddr);
}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_l_req_hash(struct sess *sp, const char *str)
+{
+ int l;
+
+ if (str == NULL)
+ str = "";
+ l = strlen(str);
+ xxxassert (sp->hash_e == sp->http->f);
+ xxxassert (sp->hash_e + l + 1 <= sp->http->e);
+ memcpy(sp->hash_e, str, l);
+ sp->hash_e[l] = '#';
+ sp->hash_e += l + 1;
+ sp->http->f += l + 1;
+}

Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_classic.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -121,22 +121,20 @@
*/

static struct objhead *
-hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
+hcl_lookup(const char *b, const char *e, struct objhead *noh)
{
struct hcl_entry *he, *he2;
struct hcl_hd *hp;
- unsigned u1, digest, kl1, kl2, kl, r;
+ unsigned u1, digest, kl, r;
int i;

CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);

- digest = crc32_2s(key1, key2);
+ digest = crc32_l(b, e - b);

u1 = digest % hcl_nhash;
hp = &hcl_head[u1];
- kl1 = strlen(key1) + 1; /* Incl '/0' */
- kl2 = strlen(key2);
- kl = kl1 + kl2;
+ kl = e - b;
he2 = NULL;

for (r = 0; r < 2; r++ ) {
@@ -151,10 +149,8 @@
continue;
if (he->digest > digest)
break;
- if (memcmp(he->key, key1, kl1))
+ if (memcmp(he->key, b, kl))
continue;
- if (memcmp(he->key + kl1, key2, kl2))
- continue;
he->refcnt++;
noh = he->oh;
UNLOCK(&hp->mtx);
@@ -190,8 +186,7 @@
noh->hashpriv = he2;

he2->key = (void*)(he2 + 1);
- memcpy(he2->key, key1, kl1);
- memcpy(he2->key + kl1, key2, kl2);
+ memcpy(he2->key, b, kl);
}
assert(he2 == NULL); /* FlexeLint */
INCOMPL();

Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -44,8 +44,8 @@

struct hsl_entry {
TAILQ_ENTRY(hsl_entry) list;
- char *key1;
- char *key2;
+ char *key;
+ int keylen;
struct objhead *obj;
unsigned refcnt;
};
@@ -73,19 +73,19 @@
*/

static struct objhead *
-hsl_lookup(const char *key1, const char *key2, struct objhead *nobj)
+hsl_lookup(const char *b, const char *e, struct objhead *nobj)
{
struct hsl_entry *he, *he2;
- int i;
+ int i, l;

+ l = e - b;
LOCK(&hsl_mutex);
TAILQ_FOREACH(he, &hsl_head, list) {
- i = strcmp(key1, he->key1);
- if (i < 0)
+ if (l > he->keylen)
continue;
- if (i > 0)
- break;
- i = strcmp(key2, he->key2);
+ if (l < he->keylen)
+ break;;
+ i = memcmp(b, he->key, l);
if (i < 0)
continue;
if (i > 0)
@@ -100,14 +100,13 @@
UNLOCK(&hsl_mutex);
return (NULL);
}
- he2 = calloc(sizeof *he2, 1);
+ he2 = calloc(sizeof *he2 + l, 1);
XXXAN(he2);
he2->obj = nobj;
he2->refcnt = 1;
- he2->key1 = strdup(key1);
- XXXAN(he2->key1);
- he2->key2 = strdup(key2);
- XXXAN(he2->key2);
+ he2->key = (void*)(he2 + 1);
+ he2->keylen = l;
+ memcpy(he2->key, b, l);
nobj->hashpriv = he2;
if (he != NULL)
TAILQ_INSERT_BEFORE(he, he2, list);
@@ -131,8 +130,6 @@
he = obj->hashpriv;
LOCK(&hsl_mutex);
if (--he->refcnt == 0) {
- free(he->key1);
- free(he->key2);
TAILQ_REMOVE(&hsl_head, he, list);
free(he);
ret = 0;

Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2007-05-09 13:55:39 UTC (rev 1397)
+++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2007-05-09 14:28:50 UTC (rev 1398)
@@ -88,6 +88,16 @@
"}\n"
"\n"
"sub vcl_hash {\n"
+ " set req.hash += req.url;\n"
+#if 1
+ " set req.hash += req.http.host;\n"
+#else
+ " if (req.http.host) {\n"
+ " set req.hash += req.http.host;\n"
+ " } else {\n"
+ " set req.hash += server.ip;\n"
+ " }\n"
+#endif
" hash;\n"
"}\n"
"\n"