Mailing List Archive

Author: des
Date: 2007-05-25 12:00:38 +0200 (Fri, 25 May 2007)
New Revision: 1474

Added:
trunk/varnish-cache/bin/varnishd/cache_synthetic.c
Modified:
trunk/varnish-cache/bin/varnishd/Makefile.am
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_backend.c
trunk/varnish-cache/bin/varnishd/cache_center.c
trunk/varnish-cache/bin/varnishd/cache_http.c
trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Add an API for synthetic objects, and use it to implement negative
caching of backend issues.

Brief summary:

- moved http_msg array from cache_response.c to cache_http.c,
introduced http_StatusMessage() lookup function

- introduced http_Put{Protocol,Status,Response} to complement
http_PrintfHeader().

- introduced SYN_ErrorPage() in a new file, cache_synthetic.c.
SYN_ErrorPage() populates the session's current object with the
specified error code and a corresponding HTML error page; it is the
caller's responsibility to ensure that the session has a suitable
object (i.e. one that doesn't already have headers or a body)

- rewrote RES_Error() to simply call SYN_ErrorPage() (with ttl = 0) and
RES_WriteObj().

- rewrote cnt_fetch() to use SYN_ErrorPage() to create a 503 page with
a TTL of 30 seconds when Fetch() fails.

- removed the call to RES_Error() in cache_backend.c; the error
trickles back up to cnt_fetch() anyway.

Comments from review:

- Memory allocation and pointer gymnastics for the header and body
are duplicated all over the place (in new and pre-existing code)
and should be centralized and hidden behind a suitable API.

- The http_*() API needs refactoring, we shouldn't need four
different functions to manipulate four different entries in the
same array.


Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am 2007-05-25 10:00:38 UTC (rev 1474)
@@ -24,6 +24,7 @@
cache_pipe.c \
cache_response.c \
cache_session.c \
+ cache_synthetic.c \
cache_vcl.c \
cache_vrt.c \
cache_vrt_acl.c \

Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2007-05-25 10:00:38 UTC (rev 1474)
@@ -365,6 +365,7 @@
void HSH_Init(void);

/* cache_http.c */
+const char *http_StatusMessage(int);
void HTTP_Init(void);
void http_ClrHeader(struct http *to);
void http_CopyHttp(struct http *to, struct http *fm);
@@ -374,6 +375,9 @@
void http_CopyResp(struct worker *w, int fd, struct http *to, struct http *fm);
void http_SetResp(struct worker *w, int fd, struct http *to, const char *proto, const char *status, const char *response);
void http_FilterHeader(struct worker *w, int fd, struct http *to, struct http *fm, unsigned how);
+void http_PutProtocol(struct worker *w, int fd, struct http *to, const char *protocol);
+void http_PutStatus(struct worker *w, int fd, struct http *to, int status);
+void http_PutResponse(struct worker *w, int fd, struct http *to, const char *response);
void http_PrintfHeader(struct worker *w, int fd, struct http *to, const char *fmt, ...);
void http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr);
void http_Setup(struct http *ht, void *space, unsigned len);
@@ -437,6 +441,9 @@
void RES_Error(struct sess *sp, int code, const char *reason);
void RES_WriteObj(struct sess *sp);

+/* cache_synthetic.c */
+void SYN_ErrorPage(struct sess *sp, int status, const char *reason, int ttl);
+
/* cache_vcl.c */
void VCL_Init(void);
void VCL_Refresh(struct VCL_conf **vcc);

Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.c 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2007-05-25 10:00:38 UTC (rev 1474)
@@ -320,7 +320,6 @@
}
usleep(100000 * n);
}
- RES_Error(sp, 503, "Backend did not respond.");
return (NULL);
}


Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c 2007-05-25 10:00:38 UTC (rev 1474)
@@ -286,27 +286,20 @@
cnt_fetch(struct sess *sp)
{

-
if (Fetch(sp)) {
- sp->obj->cacheable = 0;
- HSH_Unbusy(sp->obj);
- HSH_Deref(sp->obj);
- sp->obj = NULL;
- sp->step = STP_DONE;
- RES_Error(sp, 503, NULL);
- return (0);
- }
+ SYN_ErrorPage(sp, 503, "Error talking to backend", 30);
+ } else {
+ RFC2616_cache_policy(sp, &sp->obj->http); /* XXX -> VCL */

- RFC2616_cache_policy(sp, &sp->obj->http); /* XXX -> VCL */
+ VCL_fetch_method(sp);

- VCL_fetch_method(sp);
+ if (sp->handling == VCL_RET_ERROR)
+ INCOMPL();

- if (sp->handling == VCL_RET_ERROR)
- INCOMPL();
+ if (sp->handling == VCL_RET_PASS)
+ sp->obj->pass = 1;
+ }

- if (sp->handling == VCL_RET_PASS)
- sp->obj->pass = 1;
-
sp->obj->cacheable = 1;
if (sp->obj->objhead != NULL) {
HSH_Ref(sp->obj); /* get another, STP_DELIVER will deref */

Modified: trunk/varnish-cache/bin/varnishd/cache_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_http.c 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/cache_http.c 2007-05-25 10:00:38 UTC (rev 1474)
@@ -43,6 +43,10 @@
#include "shmlog.h"
#include "cache.h"

+#ifndef HAVE_STRLCPY
+#include <compat/strlcpy.h>
+#endif
+
#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
#include "http_headers.h"
#undef HTTPH
@@ -94,7 +98,69 @@
}

/*--------------------------------------------------------------------*/
+/* List of canonical HTTP response code names from RFC2616 */

+static struct http_msg {
+ unsigned nbr;
+ const char *txt;
+} http_msg[] = {
+ { 101, "Switching Protocols" },
+ { 200, "OK" },
+ { 201, "Created" },
+ { 202, "Accepted" },
+ { 203, "Non-Authoritative Information" },
+ { 204, "No Content" },
+ { 205, "Reset Content" },
+ { 206, "Partial Content" },
+ { 300, "Multiple Choices" },
+ { 301, "Moved Permanently" },
+ { 302, "Found" },
+ { 303, "See Other" },
+ { 304, "Not Modified" },
+ { 305, "Use Proxy" },
+ { 306, "(Unused)" },
+ { 307, "Temporary Redirect" },
+ { 400, "Bad Request" },
+ { 401, "Unauthorized" },
+ { 402, "Payment Required" },
+ { 403, "Forbidden" },
+ { 404, "Not Found" },
+ { 405, "Method Not Allowed" },
+ { 406, "Not Acceptable" },
+ { 407, "Proxy Authentication Required" },
+ { 408, "Request Timeout" },
+ { 409, "Conflict" },
+ { 410, "Gone" },
+ { 411, "Length Required" },
+ { 412, "Precondition Failed" },
+ { 413, "Request Entity Too Large" },
+ { 414, "Request-URI Too Long" },
+ { 415, "Unsupported Media Type" },
+ { 416, "Requested Range Not Satisfiable" },
+ { 417, "Expectation Failed" },
+ { 500, "Internal Server Error" },
+ { 501, "Not Implemented" },
+ { 502, "Bad Gateway" },
+ { 503, "Service Unavailable" },
+ { 504, "Gateway Timeout" },
+ { 505, "HTTP Version Not Supported" },
+ { 0, NULL }
+};
+
+const char *
+http_StatusMessage(int status)
+{
+ struct http_msg *mp;
+
+ assert(status >= 100 && status <= 999);
+ for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
+ if (mp->nbr == status)
+ return (mp->txt);
+ return ("Unknown Error");
+}
+
+/*--------------------------------------------------------------------*/
+
void
http_Setup(struct http *hp, void *space, unsigned len)
{
@@ -817,16 +883,60 @@
/*--------------------------------------------------------------------*/

void
+http_PutProtocol(struct worker *w, int fd, struct http *to, const char *protocol)
+{
+ int l;
+
+ CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ l = strlcpy(to->f, protocol, to->e - to->f);
+ xxxassert(to->f + l < to->e);
+ to->hd[HTTP_HDR_PROTO].b = to->f;
+ to->hd[HTTP_HDR_PROTO].e = to->f + l;
+ to->f += l + 1;
+ WSLH(w, HTTP_T_Protocol, fd, to, HTTP_HDR_PROTO);
+}
+
+void
+http_PutStatus(struct worker *w, int fd, struct http *to, int status)
+{
+ int l;
+
+ CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ assert(status >= 100 && status <= 999);
+ l = snprintf(to->f, to->e - to->f, "%d", status);
+ xxxassert(to->f + l < to->e);
+ to->hd[HTTP_HDR_STATUS].b = to->f;
+ to->hd[HTTP_HDR_STATUS].e = to->f + l;
+ to->f += l + 1;
+ WSLH(w, HTTP_T_Status, fd, to, HTTP_HDR_STATUS);
+}
+
+void
+http_PutResponse(struct worker *w, int fd, struct http *to, const char *response)
+{
+ int l;
+
+ CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ l = strlcpy(to->f, response, to->e - to->f);
+ xxxassert(to->f + l < to->e);
+ to->hd[HTTP_HDR_RESPONSE].b = to->f;
+ to->hd[HTTP_HDR_RESPONSE].e = to->f + l;
+ to->f += l + 1;
+ WSLH(w, HTTP_T_Response, fd, to, HTTP_HDR_RESPONSE);
+}
+
+void
http_PrintfHeader(struct worker *w, int fd, struct http *to, const char *fmt, ...)
{
va_list ap;
unsigned l, n;

CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ l = to->e - to->f;
va_start(ap, fmt);
- l = to->e - to->f;
n = vsnprintf(to->f, l, fmt, ap);
- if (n + 1 > l || to->nhd >= HTTP_HDR_MAX) {
+ va_end(ap);
+ if (n >= l || to->nhd >= HTTP_HDR_MAX) {
VSL_stats->losthdr++;
WSL(w, http2shmlog(to, HTTP_T_LostHeader), fd, "%s", to->f);
} else {
@@ -837,7 +947,6 @@
WSLH(w, HTTP_T_Header, fd, to, to->nhd);
to->nhd++;
}
- va_end(ap);
}

/*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c 2007-05-25 09:19:47 UTC (rev 1473)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c 2007-05-25 10:00:38 UTC (rev 1474)
@@ -29,8 +29,6 @@
* $Id$
*/

-#include <stdio.h> /* XXX: for NULL ?? */
-#include <string.h> /* XXX: for NULL ?? */
#include <sys/types.h>
#include <sys/time.h>

@@ -43,126 +41,27 @@
#include "cache.h"

/*--------------------------------------------------------------------*/
-/* List of canonical HTTP response code names from RFC2616 */

-static struct http_msg {
- unsigned nbr;
- const char *txt;
- const char *reason;
-} http_msg[] = {
- { 101, "Switching Protocols" },
- { 200, "OK" },
- { 201, "Created" },
- { 202, "Accepted" },
- { 203, "Non-Authoritative Information" },
- { 204, "No Content" },
- { 205, "Reset Content" },
- { 206, "Partial Content" },
- { 300, "Multiple Choices" },
- { 301, "Moved Permanently" },
- { 302, "Found" },
- { 303, "See Other" },
- { 304, "Not Modified" },
- { 305, "Use Proxy" },
- { 306, "(Unused)" },
- { 307, "Temporary Redirect" },
- { 400, "Bad Request" },
- { 401, "Unauthorized" },
- { 402, "Payment Required" },
- { 403, "Forbidden" },
- { 404, "Not Found" },
- { 405, "Method Not Allowed" },
- { 406, "Not Acceptable" },
- { 407, "Proxy Authentication Required" },
- { 408, "Request Timeout" },
- { 409, "Conflict" },
- { 410, "Gone" },
- { 411, "Length Required" },
- { 412, "Precondition Failed" },
- { 413, "Request Entity Too Large" },
- { 414, "Request-URI Too Long" },
- { 415, "Unsupported Media Type" },
- { 416, "Requested Range Not Satisfiable" },
- { 417, "Expectation Failed" },
- { 500, "Internal Server Error" },
- { 501, "Not Implemented" },
- { 502, "Bad Gateway" },
- { 503, "Service Unavailable" },
- { 504, "Gateway Timeout" },
- { 505, "HTTP Version Not Supported" },
- { 0, NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
void
RES_Error(struct sess *sp, int code, const char *reason)
{
- char buf[40];
- struct vsb *sb;
- struct http_msg *mp;
- const char *msg;

- assert(code >= 100 && code <= 999);
- CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ /* get a pristine object */
+ HSH_Prealloc(sp);
+ sp->obj = sp->wrk->nobj;
+ sp->wrk->nobj = NULL;
+ sp->obj->busy = 1;

- clock_gettime(CLOCK_REALTIME, &sp->t_resp);
+ /* synthesize error page and send it */
+ SYN_ErrorPage(sp, code, reason, 0);
+ RES_WriteObj(sp);

- msg = "Unknown error";
- for (mp = http_msg; mp->nbr != 0 && mp->nbr <= code; mp++) {
- if (mp->nbr < code)
- continue;
- if (mp->nbr > code)
- break;
- msg = mp->txt;
- if (reason == NULL)
- reason = mp->reason;
- break;
- }
- if (reason == NULL)
- reason = msg;
- AN(reason);
- AN(msg);
-
- sb = vsb_new(NULL, NULL, 0, VSB_AUTOEXTEND);
- XXXAN(sb);
-
- vsb_clear(sb);
- vsb_printf(sb, "HTTP/1.1 %03d %s\r\n", code, msg);
- TIM_format(sp->t_req.tv_sec, buf);
- vsb_printf(sb, "Date: %s\r\n", buf);
- vsb_cat(sb,
- "Server: Varnish\r\n"
- "Connection: close\r\n"
- "Content-Type: text/html; charset=iso-8859-1\r\n"
- "\r\n"
- "<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\r\n"
- "<HTML>\r\n"
- " <HEAD>\r\n");
- vsb_printf(sb, " <TITLE>%03d %s</TITLE>\r\n", code, msg);
- vsb_cat(sb,
- " </HEAD>\r\n"
- " <BODY>\r\n");
- vsb_printf(sb, " <H1>Error %03d %s</H1>\r\n", code, msg);
- vsb_printf(sb, " <P>%s</P>\r\n", reason);
- vsb_printf(sb, " <H3>Guru Meditation:</H3>\r\n");
- vsb_printf(sb, " <P>XID: %u</P>\r\n", sp->xid);
- vsb_cat(sb,
- " <I><A href=\"http://www.varnish-cache.org/\">Varnish</A></I>\r\n"
- " </BODY>\r\n"
- "</HTML>\r\n");
- vsb_finish(sb);
- WRK_Reset(sp->wrk, &sp->fd);
- sp->wrk->acct.hdrbytes += WRK_Write(sp->wrk, vsb_data(sb), vsb_len(sb));
- WRK_Flush(sp->wrk);
- WSL(sp->wrk, SLT_TxStatus, sp->id, "%d", code);
- WSL(sp->wrk, SLT_TxProtocol, sp->id, "HTTP/1.1");
- WSL(sp->wrk, SLT_TxResponse, sp->id, msg);
- vca_close_session(sp, reason);
- vsb_delete(sb);
+ /* GC the error page */
+ HSH_Unbusy(sp->obj);
+ HSH_Deref(sp->obj);
+ sp->obj = NULL;
}

-
/*--------------------------------------------------------------------*/

static void

Added: trunk/varnish-cache/bin/varnishd/cache_synthetic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_synthetic.c (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_synthetic.c 2007-05-25 10:00:38 UTC (rev 1474)
@@ -0,0 +1,142 @@
+/*-
+ * Copyright (c) 2007 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgrav <des at linpro.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id$
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <stdlib.h>
+
+#ifndef HAVE_CLOCK_GETTIME
+#include "compat/clock_gettime.h"
+#endif
+
+#include "shmlog.h"
+#include "heritage.h"
+#include "cache.h"
+
+/*
+ * Synthesize an error page. This assumes the session already has an
+ * object - if it doesn't, you need to either call HSH_Lookup(), or call
+ * HSH_Prealloc() and grab sp->obj->nobj, before calling this.
+ */
+void
+SYN_ErrorPage(struct sess *sp, int status, const char *reason, int ttl)
+{
+ struct storage *st;
+ struct object *o;
+ struct worker *w;
+ struct http *h;
+ struct vsb vsb;
+ const char *msg;
+ char date[40];
+ time_t now;
+ size_t len;
+ int fd;
+
+ assert(status >= 100 && status <= 999);
+ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ CHECK_OBJ_NOTNULL(&sp->obj->http, HTTP_MAGIC);
+ assert(sp->obj->busy > 0);
+
+ /* shortcuts */
+ w = sp->wrk;
+ fd = sp->fd;
+ o = sp->obj;
+ h = &o->http;
+ time(&now);
+
+ /* look up HTTP response */
+ msg = http_StatusMessage(status);
+ if (reason == NULL)
+ reason = msg;
+ AN(reason);
+ AN(msg);
+
+ /* populate metadata */
+ o->response = status;
+ o->valid = 1;
+ o->entered = now;
+ o->ttl = now + ttl;
+ o->last_modified = now;
+ o->xid = sp->xid;
+
+ /* allocate space for body */
+ /* XXX what if the object already has a body? */
+ st = stevedore->alloc(stevedore, 1024);
+ XXXAN(st->stevedore);
+ TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+
+ /* generate body */
+ vsb_new(&vsb, (char *)st->ptr, st->space, VSB_FIXEDLEN);
+ vsb_printf(&vsb,
+ "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+ "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n"
+ " \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
+ "<html>\n"
+ " <head>\n"
+ " <title>%03d %s</title>\n", status, msg);
+ vsb_printf(&vsb,
+ " </head>\n"
+ " <body>\n"
+ " <h1>Error %03d %s</h1>\n", status, msg);
+ vsb_printf(&vsb,
+ " <p>%s</p>\n", reason);
+ vsb_printf(&vsb,
+ " <h3>Guru Meditation:</h3>\n"
+ " <p>XID: %u</p>\n", sp->xid);
+ vsb_printf(&vsb,
+ " <address><a href=\"http://www.varnish-cache.org/\">Varnish</a></address>\n"
+ " </body>\n"
+ "</html>\n");
+ vsb_finish(&vsb);
+ o->len = st->len = vsb_len(&vsb);
+ vsb_delete(&vsb);
+
+ /* allocate space for header */
+ /* XXX what if the object already has a header? */
+ h->v = h->s = calloc(len = 1024, 1);
+ XXXAN(h->s);
+ h->e = h->s + len;
+
+ /* generate header */
+ http_ClrHeader(h);
+ http_PutProtocol(w, fd, h, "HTTP/1.0"); /* XXX */
+ http_PutStatus(w, fd, h, status);
+ http_PutResponse(w, fd, h, msg);
+ TIM_format(now, date);
+ http_PrintfHeader(w, fd, h, "Date: %s", date);
+ http_PrintfHeader(w, fd, h, "Server: Varnish");
+ http_PrintfHeader(w, fd, h, "Retry-After: %ju", (uintmax_t)ttl);
+ http_PrintfHeader(w, fd, h, "Content-Type: text/html; charset=utf-8");
+ http_PrintfHeader(w, fd, h, "Content-Length: %u", o->len);
+ /* DO NOT generate X-Varnish header, RES_WriteObj will */
+}