Mailing List Archive

Author: tfheen
Date: 2008-11-10 11:09:31 +0100 (Mon, 10 Nov 2008)
New Revision: 3370

Added:
branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc
Modified:
branches/2.0/varnish-cache/bin/varnishd/cache_ban.c
branches/2.0/varnish-cache/bin/varnishd/heritage.h
branches/2.0/varnish-cache/bin/varnishd/mgt_param.c
branches/2.0/varnish-cache/include/stat_field.h
Log:
Merge r3329, r3330: Add a boolean parameter "purge_dups"

Add a boolean parameter "purge_dups", to make it possible to mark earlier
bans of the same regexp as "gone" to save duplicate regexp checking.

Prodded to by: jodok

Add a test-case for dup purge elimination



Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2008-11-10 10:09:31 UTC (rev 3370)
@@ -49,6 +49,8 @@
#define BAN_MAGIC 0x700b08ea
VTAILQ_ENTRY(ban) list;
unsigned refcount;
+ int flags;
+#define BAN_F_GONE (1 << 0)
regex_t regexp;
char *ban;
int hash;
@@ -68,8 +70,9 @@
int
BAN_Add(struct cli *cli, const char *regexp, int hash)
{
- struct ban *b;
+ struct ban *b, *bi, *be;
char buf[512];
+ unsigned pcount;
int i;

ALLOC_OBJ(b, BAN_MAGIC);
@@ -97,8 +100,37 @@
ban_start = b;
VSL_stats->n_purge++;
VSL_stats->n_purge_add++;
+
+ if (params->purge_dups) {
+ be = VTAILQ_LAST(&ban_head, banhead);
+ be->refcount++;
+ } else
+ be = NULL;
UNLOCK(&ban_mtx);

+ if (be == NULL)
+ return (0);
+
+ /* Hunt down duplicates, and mark them as gone */
+ bi = b;
+ pcount = 0;
+ while(bi != be) {
+ bi = VTAILQ_NEXT(bi, list);
+ if (bi->flags & BAN_F_GONE)
+ continue;
+ if (b->hash != bi->hash)
+ continue;
+ if (strcmp(b->ban, bi->ban))
+ continue;
+ bi->flags |= BAN_F_GONE;
+ pcount++;
+ }
+ LOCK(&ban_mtx);
+ be->refcount--;
+ /* XXX: We should check if the tail can be removed */
+ VSL_stats->n_purge_dups += pcount;
+ UNLOCK(&ban_mtx);
+
return (0);
}

@@ -168,7 +200,8 @@
tests = 0;
for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
tests++;
- if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
+ if (!(b->flags & BAN_F_GONE) &&
+ !regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
break;
}

@@ -227,8 +260,8 @@
for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) {
if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL)
break;
- cli_out(cli, "%5u %s \"%s\"\n",
- b0->refcount,
+ cli_out(cli, "%5u %d %s \"%s\"\n",
+ b0->refcount, b0->flags,
b0->hash ? "hash" : "url ",
b0->ban);
}

Modified: branches/2.0/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/heritage.h 2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/heritage.h 2008-11-10 10:09:31 UTC (rev 3370)
@@ -183,6 +183,9 @@
/* Amount of time to sleep when running out of file
descriptors. In msecs */
unsigned accept_fd_holdoff;
+
+ /* Get rid of duplicate purges */
+ unsigned purge_dups;
};

extern volatile struct params *params;

Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2008-11-10 10:09:31 UTC (rev 3370)
@@ -817,6 +817,10 @@
"The TTL assigned to the synthesized error pages\n",
0,
"0", "seconds" },
+ { "purge_dups", tweak_bool, &master.purge_dups, 0, 0,
+ "Detect and eliminate duplicate purges.\n",
+ 0,
+ "off", "bool" },
{ NULL, NULL, NULL }
};


Copied: branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc (from rev 3330, trunk/varnish-cache/bin/varnishtest/c00019.vtc)
===================================================================
--- branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc (rev 0)
+++ branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc 2008-11-10 10:09:31 UTC (rev 3370)
@@ -0,0 +1,83 @@
+# $Id$
+
+test "Check purge counters and duplicate purge elimination"
+
+server s1 {
+ rxreq
+ txresp -hdr "foo: 1" -body "foo1"
+ rxreq
+ txresp -hdr "foo: 2" -body "foo2"
+ rxreq
+ txresp -hdr "foo: 3" -body "foo3"
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "purge.url FOO"
+
+# There is one "magic" purge from boot
+varnish v1 -expect n_purge_add == 2
+varnish v1 -cliok "purge.list"
+
+# Our fetch is not affected by the purge
+# as the FOO-purge was preexisting
+client c1 {
+ txreq -url /FOO
+ rxresp
+ expect resp.http.foo == 1
+} -run
+
+varnish v1 -cliok "purge.list"
+varnish v1 -expect n_purge_obj_test == 0
+varnish v1 -expect n_purge_re_test == 0
+
+# Add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 3
+varnish v1 -cliok "purge.list"
+
+# The cached object will be purged, and a new
+# fetched from the backend
+client c1 {
+ txreq -url /FOO
+ rxresp
+ expect resp.http.foo == 2
+} -run
+
+varnish v1 -expect n_purge_obj_test == 1
+varnish v1 -expect n_purge_re_test == 1
+varnish v1 -cliok "purge.list"
+
+# Fetch the cached copy, just for grins
+client c1 {
+ txreq -url /FOO
+ rxresp
+ expect resp.http.foo == 2
+} -run
+
+
+# Now add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 4
+
+# Enable dup removal of purges
+varnish v1 -cliok "param.set purge_dups on"
+
+# This should incapacitate the to previous FOO purges.
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 5
+varnish v1 -expect n_purge_dups == 3
+varnish v1 -cliok "purge.list"
+
+# And we should get a fresh object from backend
+client c1 {
+ txreq -url /FOO
+ rxresp
+ expect resp.http.foo == 3
+} -run
+
+# With only two objects having ever been compared
+varnish v1 -expect n_purge_obj_test == 2
+varnish v1 -expect n_purge_re_test == 2
+varnish v1 -cliok "purge.list"
+

Modified: branches/2.0/varnish-cache/include/stat_field.h
===================================================================
--- branches/2.0/varnish-cache/include/stat_field.h 2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/include/stat_field.h 2008-11-10 10:09:31 UTC (rev 3370)
@@ -124,3 +124,4 @@
MAC_STAT(n_purge_retire, uint64_t, 'a', "N old purges deleted")
MAC_STAT(n_purge_obj_test, uint64_t, 'a', "N objects tested")
MAC_STAT(n_purge_re_test, uint64_t, 'a', "N regexps tested against")
+MAC_STAT(n_purge_dups, uint64_t, 'a', "N duplicate purges removed")