Mailing List Archive

[Bug 66030] New: Fail on changment of status with load balancer and hcheck of proxy module
https://bz.apache.org/bugzilla/show_bug.cgi?id=66030

Bug ID: 66030
Summary: Fail on changment of status with load balancer and
hcheck of proxy module
Product: Apache httpd-2
Version: 2.4.46
Hardware: Other
OS: Linux
Status: NEW
Severity: blocker
Priority: P2
Component: mod_proxy_hcheck
Assignee: bugs@httpd.apache.org
Reporter: henry.galvez@intm.fr
Target Milestone: ---

Created attachment 38263
--> https://bz.apache.org/bugzilla/attachment.cgi?id=38263&action=edit
Trace logs

Hi to all

We have a host that evaluates access with a Balancer and a HealthCheck.

In version 2.4.43, if the HealthCheck's status is 1 and after a certain time it
returns to 0, everything goes back to normal.

But from version 2.4.46 until version 2.4.52, when HealthCheck returns to
status 0, it continues to throw the following error: "All Workers Are In Error
State" and forces the restart of the HTTPD service.

Configuration example:

```config
#################################################################
## httpd.conf ##
##################################################################

ServerRoot /usr/local/apache2

LoadModule autoindex_module
/usr/local/apache2/modules/mod_autoindex.so
LoadModule unique_id_module
/usr/local/apache2/modules/mod_unique_id.so
#LoadModule mpm_worker_module
/usr/local/apache2/modules/mod_mpm_worker.so
LoadModule authn_core_module
/usr/local/apache2/modules/mod_authn_core.so
LoadModule authz_core_module
/usr/local/apache2/modules/mod_authz_core.so
LoadModule auth_basic_module
/usr/local/apache2/modules/mod_auth_basic.so
LoadModule authn_file_module
/usr/local/apache2/modules/mod_authn_file.so
LoadModule authz_user_module
/usr/local/apache2/modules/mod_authz_user.so
LoadModule mpm_event_module
/usr/local/apache2/modules/mod_mpm_event.so
LoadModule headers_module
/usr/local/apache2/modules/mod_headers.so
LoadModule authz_host_module
/usr/local/apache2/modules/mod_authz_host.so
LoadModule deflate_module
/usr/local/apache2/modules/mod_deflate.so
LoadModule setenvif_module
/usr/local/apache2/modules/mod_setenvif.so
LoadModule userdir_module
/usr/local/apache2/modules/mod_userdir.so
LoadModule dir_module /usr/local/apache2/modules/mod_dir.so
LoadModule expires_module
/usr/local/apache2/modules/mod_expires.so
LoadModule rewrite_module
/usr/local/apache2/modules/mod_rewrite.so
LoadModule alias_module /usr/local/apache2/modules/mod_alias.so
LoadModule mime_module /usr/local/apache2/modules/mod_mime.so
LoadModule status_module
/usr/local/apache2/modules/mod_status.so
LoadModule proxy_module /usr/local/apache2/modules/mod_proxy.so
LoadModule proxy_http_module
/usr/local/apache2/modules/mod_proxy_http.so
LoadModule cache_module /usr/local/apache2/modules/mod_cache.so
LoadModule proxy_ajp_module
/usr/local/apache2/modules/mod_proxy_ajp.so
LoadModule ssl_module /usr/local/apache2/modules/mod_ssl.so
LoadModule proxy_balancer_module
/usr/local/apache2/modules/mod_proxy_balancer.so
LoadModule slotmem_shm_module
/usr/local/apache2/modules/mod_slotmem_shm.so
LoadModule socache_shmcb_module
/usr/local/apache2/modules/mod_socache_shmcb.so
LoadModule lbmethod_byrequests_module
/usr/local/apache2/modules/mod_lbmethod_byrequests.so
LoadModule info_module /usr/local/apache2/modules/mod_info.so
LoadModule proxy_hcheck_module
/usr/local/apache2/modules/mod_proxy_hcheck.so
LoadModule unixd_module /usr/local/apache2/modules/mod_unixd.so
LoadModule log_config_module
/usr/local/apache2/modules/mod_log_config.so
LoadModule watchdog_module
/usr/local/apache2/modules/mod_watchdog.so

<IfModule proxy_hcheck_module>
LogLevel proxy_hcheck:trace8 proxy:trace8
</IfModule>

Timeout 1800
KeepAlive on
MaxRequestWorkers 8192
ServerLimit 128
ThreadsPerChild 64
MaxRequestsPerChild 1000
StartServers 3
MinSpareThreads 5
MaxSpareThreads 250
MaxKeepAliveRequests 250
KeepAliveTimeout 5
AsyncRequestWorkerFactor 2

userdir disabled root

TypesConfig /usr/local/apache2/conf/mime.types

AddEncoding x-compress Z
AddEncoding x-gzip gz tgz

############################################
# Definitions
############################################

Listen ######:443

ServerName ######

ServerTokens Prod
DeflateFilterNote ratio

# Serveur HTTPD
DocumentRoot /usr/local/apache2/htdocs/
DirectoryIndex index.html index.htm index.jsp

# Filtering accessLog to delete images, CSS and JavaScript
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla|css|js|pdf|ico)$ dontlog

RequestHeader setifempty Correlation-ID %{UNIQUE_ID}e
SetEnvIf Correlation-ID "(.*)" MyCorrelationId=$1
Header setifempty Correlation-ID "%{MyCorrelationId}e"

# Recovery of the original applicant only (X-Forwarded-For = host1, host2,
host3... If we pass through several layers)
RequestHeader setIfEmpty X-Forwarded-For "%{REMOTE_ADDR}s"
RequestHeader edit X-Forwarded-For ^([^,]*),?.*$ $1

# Logs management
LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\"
(%{ratio}n) \"%{User-Agent}i\" %D %{Correlation-ID}i %H" combined
ErrorLogFormat "[%t] [%m:%l] [pid %P:tid %T] [client\ %a] [referer\
%{Referer}i] [correlation_id\ %{Correlation-ID}i] %M -BT: %{error==}B -
%{oserror==}B"

PidFile /usr/local/apache2/httpd.pid
Mutex file:/usr/local/apache2 default
DefaultRuntimeDir /usr/local/apache2

ScoreBoardFile /usr/local/apache2/logs/httpd.scoreboard
CustomLog "|| /usr/local/apache2/bin/rotatelogs -l
/usr/local/apache2/logs/access_log.%Y-%m-%d-%H_%M_%S 3600" combined
env=!dontlog
ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l
/usr/local/apache2/logs/error_log.%Y-%m-%d-%H_%M_%S 60"

LogLevel debug

# Rewrite to redirect root URLs to the application URL
RewriteEngine on

RewriteRule "^/services/param/([0-9]+)/date/(.*)" "/services/"

RewriteCond %{HTTPS} !=on
RewriteRule ^/(.*)
[[https://%{HTTP_HOST}/$1]https://%{HTTP_HOST}/$1]https://%{HTTP_HOST}/$1 [R,L]

# Parametering for SSL
SSLRandomSeed startup builtin
AddType application/x-x509-ca-cert .crt
AddType application/x-pkcs7-crl .crl
SSLPassPhraseDialog builtin
SSLSessionCache "shmcb:/usr/local/apache2/logs/ssl_scache(512000)"
Mutex default
SSLEngine on

SSLCipherSuite ECDHE-#########-SHA256
# Active only TLS v1.1 & v1.2
SSLProtocol All -SSLv3
SSLCertificateFile /usr/local/apache2/secret/cert_pki_ent_HP.pem
SSLCertificateKeyFile /usr/local/apache2/secret/cert_pki_ent_HP.key
SSLCertificateChainFile /usr/local/apache2/secret/cert_pki_ent_HP.cer
SSLHonorCipherOrder off

SSLProxyEngine on

SetEnvIf User-Agent ".*MSIE.*" \
nokeepalive ssl-unclean-shutdown \
downgrade-1.0 force-response-1.0

Customlog "||/usr/local/apache2/bin/rotatelogs -l
//usr/local/apache2/logs/ssl_request_log.%Y-%m-%d-%H_%M_%S 3600" "%t %h
%{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"

# Must be outside <location></location>
Options +FollowSymLinks

############################################
# Customer Cache Treatment
############################################
RequestHeader edit "If-None-Match" "^\"(.*)-gzip\"$" "\"$1\""
Header edit "ETag" "^\"(.*[^g][^z][^i][^p])\"$" "\"$1-gzip\""
ExpiresActive On
ExpiresDefault "access plus 0 minutes"
ExpiresByType text/html "access plus 0 minutes"
ExpiresByType image/gif "access plus 24 hours"
ExpiresByType image/jpeg "access plus 24 hours"
ExpiresByType image/png "access plus 24 hours"
ExpiresByType text/css "access plus 24 hours"
ExpiresByType text/javascript "access plus 24 hours"
ExpiresByType application/javascript "access plus 24 hours"
ExpiresByType image/svg+xml "access plus 24 hours"

<Location />
SetOutputFilter DEFLATE
BrowserMatch ^Mozilla/4 gzip-only-text/html
BrowserMatch ^Mozilla/4\.0[678] no-gzip
BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png|swf|fla)$ no-gzip dont-vary
Header append Vary User-Agent env=!dont-vary
</Location>

# Rewrite to redirect root URLs to the application URL
Header edit Set-Cookie "^(API_JSESSIONID=.*); .*" "$1; domain=.example.com;
HttpOnly; secure"
Header edit Set-Cookie (XSRF-TOKEN=.*);.* "$1; path=/; domain=.example.com;
secure"

ExtendedStatus On

# The server returns Server: Apache
ServerTokens productOnly
# Returns an error code 405 (unauthorized method) to the client (mod_proxy)
TraceEnable off

ServerSignature off
HostnameLookups off

# Security Restriction: Cache Apache and prevents the use of trace mode
RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
RewriteRule .* - [F]

# Parameters for validation Geomobile Client Certificate
SSLVerifyDepth 2
SSLCACertificateFile /usr/local/apache2/secret/chain_bundle.pem
SSLVerifyClient none

<VirtualHost *:443>

SSLEngine on

ServerName api.example.com
ServerAlias api.example.com

<IfModule proxy_hcheck_module>
LogLevel proxy_hcheck:trace8 proxy:trace8
</IfModule>

# Server HTTPD
DocumentRoot /usr/local/apache2/htdocs/test

SetEnvIf Origin
"http(s)?://(www\.)?(admin.example.com:443|home.example.com:443|warnings.example.com:443|files.example.com:443|hook.example.com:443|optional.example.com:443|review.example.com:443|api.example.com:443|cluster.example.com:54657|cluster.example.com:54656|cluster.example.com:54658|cluster.example.com:54655|cluster.example.com:54659|cluster.example.com:54660|cluster.example.com:54661)$"
AccessControlAllowOrigin=$0
Header set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e
env=AccessControlAllowOrigin
Header set Access-Control-Allow-Credentials "true"
# CSRF
RequestHeader unset Access-Control-Request-Headers
Header set Access-Control-Allow-Headers "X-XSRF-TOKEN, content-type"
Header set Access-Control-Expose-Headers Circuit-Breakers-On-Error

CustomLog "|| /usr/local/apache2/bin/rotatelogs -p
'/usr/local/apache2/outils/MANAGE_LOGS.sh' -l
//usr/local/apache2/logs/access_test_log.%Y-%m-%d-%H_%M_%S 3600" combined
env=!dontlog
ErrorLog "|| /usr/local/apache2/bin/rotatelogs -l
//usr/local/apache2/logs/error_test_log.%Y-%m-%d-%H_%M_%S 60"

ProxyHCExpr reqok {%{REQUEST_STATUS} =~ /^[2]/}

<Proxy balancer://application-files/>
BalancerMember http://server-one:9382 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9382 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>

<Proxy balancer://application-db/>
BalancerMember http://server-one:9385 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9385 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>

<Proxy balancer://application-jobs/>
BalancerMember http://server-one:9384 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
BalancerMember http://server-two:9384 ttl=30 connectiontimeout=2
Keepalive=On hcmethod=HEAD hcexpr=reqok hcinterval=10 hcuri=/api/health
</Proxy>

# Balancer manager
ProxyPass /balancer-manager !
<Location /balancer-manager>
SetHandler balancer-manager
Require host server-one.example.com server-two.example.com server-one
server-two
</Location>

ProxyPass /config.api.json !
ProxyPass /config-extra.api.json !
ProxyPass /version.json !
ProxyPassMatch /management.* !

ProxyPass /services/ balancer://application-files/services/
ProxyPassReverse /services/ balancer://application-files/services/

ProxyPass /db_services/ balancer://application-db/db_services/
ProxyPassReverse /db_services/ balancer://application-db/db_services/

ProxyPass /jobs/ balancer://application-jobs/jobs/
ProxyPassReverse /jobs/ balancer://application-jobs/jobs/

</VirtualHost>
```
In the logs I can't find the reason stopping the performance of the HealthCheck
and the Balancer.

I try take help from the mail-list, but no one help me.
I think it's a bug

--
You are receiving this mail because:
You are the assignee for the bug.
---------------------------------------------------------------------
To unsubscribe, e-mail: bugs-unsubscribe@httpd.apache.org
For additional commands, e-mail: bugs-help@httpd.apache.org