fix(timeout): graceful timeout in peregrine (#112)

* Also muting /_status logging and timeout errors
uc-cdis · Feb 25, 2019 · c36d7ac · c36d7ac
1 parent 47a23a8
commit c36d7ac
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 3 deletions.
diff --git a/deployment/nginx/uwsgi.conf b/deployment/nginx/uwsgi.conf
@@ -7,13 +7,30 @@ server {
         uwsgi_param REMOTE_REQID $http_x_reqid if_not_empty;
         uwsgi_param REMOTE_SESSIONID $http_x_sessionid if_not_empty;
         uwsgi_param REMOTE_VISITORID $http_x_visitorid if_not_empty;
+        uwsgi_param GEN3_REQUEST_TIMESTAMP $msec;
+        uwsgi_param GEN3_TIMEOUT_SECONDS '45';
 
         include uwsgi_params;
         uwsgi_pass unix:/var/run/gen3/uwsgi.sock;
         uwsgi_read_timeout 45s;
         uwsgi_send_timeout 45s;
     }
 
+    location /_status {
+        include uwsgi_params;
+        uwsgi_param GEN3_REQUEST_TIMESTAMP $msec;
+        uwsgi_param GEN3_TIMEOUT_SECONDS $arg_timeout;
+        uwsgi_pass unix:/var/run/gen3/uwsgi.sock;
+        uwsgi_read_timeout 45s;
+        uwsgi_ignore_client_abort on;
+        access_log off;
+    }
+
+    error_page 502 /502.html;
+    location /502.html {
+        return 504 '{"error": {"Request Timeout or Service Unavailable"}}';
+    }
+
     error_page 504 /504.html;
     location /504.html {
         return 504 '{"error": {"Request Timeout"}}';

diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini
@@ -5,7 +5,13 @@ buffer-size = 32768
 chmod-socket = 666
 master = true
 processes = 2
-harakiki-verbose = 20
+harakiri-verbose = true
+harakiri = 45
+http-timeout = 45
+socket-timeout = 45
+worker-reload-mercy = 45
+reload-mercy = 45
+mule-reload-mercy = 45
 wsgi-file=/var/www/peregrine/wsgi.py
 plugins = python
 vacuum = true

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -19,7 +19,6 @@ envelopes==0.4
 # dependencies of fence
 pyasn1-modules==0.0.11
 urllib3==1.23
--e git+https://[email protected]/uc-cdis/[email protected]#egg=cdispyutils
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=cirrus-0.1.5
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=fence
 # userdatamodel is required by fence

diff --git a/peregrine/api.py b/peregrine/api.py
@@ -12,6 +12,7 @@
 import datamodelutils
 from dictionaryutils import DataDictionary, dictionary as dict_init
 from cdispyutils.log import get_handler
+from cdispyutils.uwsgi import setup_user_harakiri
 
 import peregrine
 from peregrine import dictionary
@@ -109,6 +110,9 @@ def app_init(app):
     app.logger.info('Initializing app')
     dictionary_init(app)
 
+    if app.config.get("USE_USER_HARAKIRI", True):
+        setup_user_harakiri(app)
+
     app_register_blueprints(app)
     app_register_duplicate_blueprints(app)
 

diff --git a/requirements.txt b/requirements.txt
@@ -35,7 +35,7 @@ Flask-SQLAlchemy-Session==1.1
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=gdcdatamodel
 -e git+https://[email protected]/NCI-GDC/psqlgraph.git@5cddf49dd03a25bd4e553161d7ad7b9a6fe0ac0d#egg=psqlgraph
 -e git+https://[email protected]/NCI-GDC/cdisutils.git@8a8e599fdab5ade9bd8c586132d974a102e5d72d#egg=cdisutils
--e git+https://[email protected]/uc-cdis/[email protected]#egg=cdispyutils
+cdispyutils==0.2.12
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=storageclient
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=dictionaryutils
 -e git+https://[email protected]/uc-cdis/[email protected]#egg=datamodelutils