From c4bdb04795aa3fde21869d7e68d27030240258c7 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Thu, 24 Aug 2023 13:09:23 +0200 Subject: [PATCH 1/3] Use X-Accel-Redirect for serving ajax-proxy request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uMap allows to use remote URL as data sources, but those URLs are not always CORS open, so this is why there is this "ajax-proxy" feature, where the URL is passed to the backend. Additionally, there is a caching feature, which duration is configurable through frontend settings. Valid values are: disabled, 5 min, 1 hour, 1 day. Initially, I wanted this to be totally handled by Nginx, but I never found a wayt to set the proxy_cache_valid value from a query string. Since then, at least in OSM France servers, the ajax-proxy is still handled by a Django view, which then opens the remote URL and transfert the data. This is not optimal. And I suppose this is what is causing hicups on the OSM France servers lately. This PR provides a mix option, where python deals with validating the URL and parsing the TTL parameter, and then it passes the hand to nginx which will serve the remote content. So, roughtly: - the client calls /ajax-proxy/?url=xxx&ttl=300 - python will validate the URL (not internal calls…) - if UMAP_SENDFILE_HEADER is set, then the python returns an empty response with the path /proxy/http://url plus it will set the cache ttl through the header X-Accel-Expires - this /proxy/ location is then handled by nginx --- docs/ubuntu.md | 45 ++++++++++++++++++++++++++++----------------- umap/views.py | 17 ++++++++++++----- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/docs/ubuntu.md b/docs/ubuntu.md index 570c1c57..83baaa27 100644 --- a/docs/ubuntu.md +++ b/docs/ubuntu.md @@ -332,6 +332,7 @@ And then add this new location in your nginx config (before the `/` location): alias /path/to/umap/var/data/; } + ### Configure ajax proxy cache In Nginx: @@ -341,28 +342,38 @@ In Nginx: proxy_cache_path /tmp/nginx_ajax_proxy_cache levels=1:2 keys_zone=ajax_proxy:10m inactive=60m; proxy_cache_key "$args"; -- add this location (before the `/` location): +- add those locations (before the `/` location): - location /ajax-proxy/ { - valid_referers server_names; - if ($invalid_referer) { - return 400; - } + location ~ ^/proxy/(.*) { + internal; add_header X-Proxy-Cache $upstream_cache_status always; proxy_cache ajax_proxy; proxy_cache_valid 1m; # Default. Umap will override using X-Accel-Expires - gzip on; - gzip_proxied any; - gzip_types - application/vnd.google-earth.kml+xml - application/geo+json - application/json - application/javascript - text/xml - application/xml; - uwsgi_pass umap; - include /srv/umap/uwsgi_params; + set $target_url $1; + # URL is encoded, so we need a few hack to clean it back. + if ( $target_url ~ (.+)%3A%2F%2F(.+) ){ # fix :// between scheme and destination + set $target_url $1://$2; + } + if ( $target_url ~ (.+?)%3A(.*) ){ # fix : between destination and port + set $target_url $1:$2; + } + if ( $target_url ~ (.+?)%2F(.*) ){ # fix / after port, the rest will be decoded by proxy_pass + set $target_url $1/$2; + } + add_header X-Proxy-Target $target_url; # For debugging + resolver 8.8.8.8; + proxy_read_timeout 10s; + proxy_connect_timeout 5s; + proxy_pass $target_url; + proxy_intercept_errors on; + error_page 301 302 307 = @handle_proxy_redirect; } + location @handle_proxy_redirect { + resolver 8.8.8.8; + set $saved_redirect_location '$upstream_http_location'; + proxy_pass $saved_redirect_location; + } + diff --git a/umap/views.py b/umap/views.py index 6e6b905e..b44cd2f4 100644 --- a/umap/views.py +++ b/umap/views.py @@ -351,6 +351,17 @@ class AjaxProxy(View): url = validate_url(self.request) except AssertionError: return HttpResponseBadRequest() + try: + ttl = int(self.request.GET.get("ttl")) + except (TypeError, ValueError): + ttl = None + if getattr(settings, "UMAP_XSENDFILE_HEADER", None): + response = HttpResponse() + response[settings.UMAP_XSENDFILE_HEADER] = f"/proxy/{url}" + if ttl: + response["X-Accel-Expires"] = ttl + return response + headers = {"User-Agent": "uMapProxy +http://wiki.openstreetmap.org/wiki/UMap"} request = Request(url, headers=headers) opener = build_opener() @@ -375,11 +386,7 @@ class AjaxProxy(View): # Quick hack to prevent Django from adding a Vary: Cookie header self.request.session.accessed = False response = HttpResponse(content, status=status_code, content_type=mimetype) - try: - ttl = int(self.request.GET.get("ttl")) - except (TypeError, ValueError): - pass - else: + if ttl: response["X-Accel-Expires"] = ttl return response From 7210b1ddb9bd671c988b9eeaa07b5d0b991a88e9 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Thu, 24 Aug 2023 17:48:41 +0200 Subject: [PATCH 2/3] Remove useless resolver from nginx config in doc --- docs/ubuntu.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ubuntu.md b/docs/ubuntu.md index 83baaa27..634f710d 100644 --- a/docs/ubuntu.md +++ b/docs/ubuntu.md @@ -361,7 +361,6 @@ In Nginx: set $target_url $1/$2; } add_header X-Proxy-Target $target_url; # For debugging - resolver 8.8.8.8; proxy_read_timeout 10s; proxy_connect_timeout 5s; proxy_pass $target_url; @@ -369,7 +368,6 @@ In Nginx: error_page 301 302 307 = @handle_proxy_redirect; } location @handle_proxy_redirect { - resolver 8.8.8.8; set $saved_redirect_location '$upstream_http_location'; proxy_pass $saved_redirect_location; } From 2904bcc61790b74cf280bd1fbd7e77cbcb263318 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 28 Aug 2023 16:36:50 +0200 Subject: [PATCH 3/3] More documentation about "ajax-proxy" --- docs/ubuntu.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/ubuntu.md b/docs/ubuntu.md index 634f710d..8b7cd008 100644 --- a/docs/ubuntu.md +++ b/docs/ubuntu.md @@ -335,6 +335,27 @@ And then add this new location in your nginx config (before the `/` location): ### Configure ajax proxy cache +uMap allows to use remote URL as data sources, but those URLs are not always +CORS open, so this is why there is this "ajax-proxy" feature, where the URL is +passed to the backend. + +Additionally, there is a caching feature, which duration is configurable through +frontend settings. Valid values are: disabled, 5 min, 1 hour, 1 day. + +This configuration provides a mix option, where python deals with validating the +URL and parsing the TTL parameter, and then it passes the hand to nginx which +will serve the remote content. + +So, roughly: + +- the client calls `/ajax-proxy/?url=xxx&ttl=300` +- python will validate the URL (not internal calls…) +- if `UMAP_XSENDFILE_HEADER` is set, then the python returns an empty response + with the path `/proxy/http://url` plus it will set the cache TTL through the + header `X-Accel-Expires` +- this `/proxy/` location is then handled by nginx + + In Nginx: - add the proxy cache