From 9d752ea306749c1b9b37fddf79762d6037c0e521 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Mon, 22 May 2023 17:47:04 -0400 Subject: [PATCH 1/3] Add a very basic `/stats/` JSON view Will be useful to feed munin for instance. --- umap/tests/test_views.py | 86 ++++++++++++++++++++++++++-------------- umap/urls.py | 13 +++--- umap/views.py | 26 +++++++++--- 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/umap/tests/test_views.py b/umap/tests/test_views.py index 41232fa2..228d4dd5 100644 --- a/umap/tests/test_views.py +++ b/umap/tests/test_views.py @@ -1,4 +1,6 @@ +import json import socket +from datetime import date import pytest from django.conf import settings @@ -11,12 +13,12 @@ from umap.views import validate_url def get(target="http://osm.org/georss.xml", verb="get", **kwargs): defaults = { - 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest', - 'HTTP_REFERER': '%s/path/' % settings.SITE_URL + "HTTP_X_REQUESTED_WITH": "XMLHttpRequest", + "HTTP_REFERER": "%s/path/" % settings.SITE_URL, } defaults.update(kwargs) func = getattr(RequestFactory(**defaults), verb) - return func('/', {'url': target}) + return func("/", {"url": target}) def test_good_request_passes(): @@ -70,67 +72,91 @@ def test_unkown_domain_raises(): def test_valid_proxy_request(client): - url = reverse('ajax-proxy') - params = {'url': 'http://example.org'} + url = reverse("ajax-proxy") + params = {"url": "http://example.org"} headers = { - 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest', - 'HTTP_REFERER': settings.SITE_URL + "HTTP_X_REQUESTED_WITH": "XMLHttpRequest", + "HTTP_REFERER": settings.SITE_URL, } response = client.get(url, params, **headers) assert response.status_code == 200 - assert 'Example Domain' in response.content.decode() - assert 'Cookie' not in response['Vary'] + assert "Example Domain" in response.content.decode() + assert "Cookie" not in response["Vary"] def test_valid_proxy_request_with_ttl(client): - url = reverse('ajax-proxy') - params = {'url': 'http://example.org', 'ttl': 3600} + url = reverse("ajax-proxy") + params = {"url": "http://example.org", "ttl": 3600} headers = { - 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest', - 'HTTP_REFERER': settings.SITE_URL + "HTTP_X_REQUESTED_WITH": "XMLHttpRequest", + "HTTP_REFERER": settings.SITE_URL, } response = client.get(url, params, **headers) assert response.status_code == 200 - assert 'Example Domain' in response.content.decode() - assert 'Cookie' not in response['Vary'] - assert response['X-Accel-Expires'] == '3600' + assert "Example Domain" in response.content.decode() + assert "Cookie" not in response["Vary"] + assert response["X-Accel-Expires"] == "3600" def test_valid_proxy_request_with_invalid_ttl(client): - url = reverse('ajax-proxy') - params = {'url': 'http://example.org', 'ttl': 'invalid'} + url = reverse("ajax-proxy") + params = {"url": "http://example.org", "ttl": "invalid"} headers = { - 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest', - 'HTTP_REFERER': settings.SITE_URL + "HTTP_X_REQUESTED_WITH": "XMLHttpRequest", + "HTTP_REFERER": settings.SITE_URL, } response = client.get(url, params, **headers) assert response.status_code == 200 - assert 'Example Domain' in response.content.decode() - assert 'Cookie' not in response['Vary'] - assert 'X-Accel-Expires' not in response + assert "Example Domain" in response.content.decode() + assert "Cookie" not in response["Vary"] + assert "X-Accel-Expires" not in response @pytest.mark.django_db def test_login_does_not_contain_form_if_not_enabled(client, settings): settings.ENABLE_ACCOUNT_LOGIN = False - response = client.get(reverse('login')) - assert 'username' not in response.content.decode() + response = client.get(reverse("login")) + assert "username" not in response.content.decode() @pytest.mark.django_db def test_login_contains_form_if_enabled(client, settings): settings.ENABLE_ACCOUNT_LOGIN = True - response = client.get(reverse('login')) - assert 'username' in response.content.decode() + response = client.get(reverse("login")) + assert "username" in response.content.decode() @pytest.mark.django_db def test_can_login_with_username_and_password_if_enabled(client, settings): settings.ENABLE_ACCOUNT_LOGIN = True User = get_user_model() - user = User.objects.create(username='test') - user.set_password('test') + user = User.objects.create(username="test") + user.set_password("test") user.save() - client.post(reverse('login'), {'username': 'test', 'password': 'test'}) + client.post(reverse("login"), {"username": "test", "password": "test"}) user = get_user(client) assert user.is_authenticated + + +@pytest.mark.django_db +def test_stats_empty(client): + response = client.get(reverse("stats")) + assert json.loads(response.content.decode()) == { + "maps_active_last_week_count": 0, + "maps_count": 0, + "users_active_last_week_count": 0, + "users_count": 0, + } + + +@pytest.mark.django_db +def test_stats_basic(client, map, datalayer): + map.owner.last_login = date.today() + map.owner.save() + response = client.get(reverse("stats")) + assert json.loads(response.content.decode()) == { + "maps_active_last_week_count": 1, + "maps_count": 1, + "users_active_last_week_count": 1, + "users_count": 1, + } diff --git a/umap/urls.py b/umap/urls.py index 1c35bdb7..0154aaf1 100644 --- a/umap/urls.py +++ b/umap/urls.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.urls import include, re_path +from django.urls import include, path, re_path from django.conf.urls.i18n import i18n_patterns from django.conf.urls.static import static from django.contrib import admin @@ -41,9 +41,7 @@ urlpatterns = [ ] i18n_urls = [ - re_path( - r"^login/$", jsonize_view(auth_views.LoginView.as_view()), name="login" - ), + re_path(r"^login/$", jsonize_view(auth_views.LoginView.as_view()), name="login"), re_path( r"^login/popup/end/$", views.LoginPopupEnd.as_view(), name="login_popup_end" ), @@ -96,9 +94,9 @@ i18n_urls += decorated_patterns( i18n_urls += decorated_patterns( [login_required], re_path( - r'^map/(?P[\d]+)/star/$', + r"^map/(?P[\d]+)/star/$", views.ToggleMapStarStatus.as_view(), - name='map_star' + name="map_star", ), ) i18n_urls += decorated_patterns( @@ -151,10 +149,11 @@ urlpatterns += i18n_patterns( ), re_path(r"^search/$", views.search, name="search"), re_path(r"^about/$", views.about, name="about"), - re_path(r"^user/(?P.+)/stars/$", views.user_stars, name='user_stars'), + re_path(r"^user/(?P.+)/stars/$", views.user_stars, name="user_stars"), re_path(r"^user/(?P.+)/$", views.user_maps, name="user_maps"), re_path(r"", include(i18n_urls)), ) +urlpatterns += (path("stats/", views.stats, name="stats"),) if settings.DEBUG and settings.MEDIA_ROOT: urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/umap/views.py b/umap/views.py index 877e9501..16018c49 100644 --- a/umap/views.py +++ b/umap/views.py @@ -3,6 +3,7 @@ import mimetypes import os import re import socket +from datetime import date, timedelta from pathlib import Path from django.conf import settings @@ -10,7 +11,7 @@ from django.contrib import messages from django.contrib.auth import logout as do_logout from django.contrib.auth import get_user_model from django.contrib.gis.measure import D -from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector +from django.contrib.postgres.search import SearchQuery, SearchVector from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.core.signing import BadSignature, Signer from django.core.validators import URLValidator, ValidationError @@ -214,7 +215,7 @@ class Search(TemplateView, PaginatorMixin): q, config=settings.UMAP_SEARCH_CONFIGURATION, search_type="websearch" ) qs = Map.objects.annotate(search=vector).filter(search=query) - qs = qs.filter(share_status=Map.PUBLIC).order_by('-modified_at') + qs = qs.filter(share_status=Map.PUBLIC).order_by("-modified_at") results = self.paginate(qs) kwargs.update({"maps": results, "q": q}) return kwargs @@ -381,7 +382,7 @@ class MapDetailMixin: "allowEdit": self.is_edit_allowed(), "default_iconUrl": "%sumap/img/marker.png" % settings.STATIC_URL, # noqa "umap_id": self.get_umap_id(), - 'starred': self.is_starred(), + "starred": self.is_starred(), "licences": dict((l.name, l.json) for l in Licence.objects.all()), "edit_statuses": [(i, str(label)) for i, label in Map.EDIT_STATUS], "share_statuses": [ @@ -663,9 +664,8 @@ class MapClone(PermissionsMixin, View): class ToggleMapStarStatus(View): - def post(self, *args, **kwargs): - map_inst = get_object_or_404(Map, pk=kwargs['map_id']) + map_inst = get_object_or_404(Map, pk=kwargs["map_id"]) qs = Star.objects.filter(map=map_inst, by=self.request.user) if qs.exists(): qs.delete() @@ -852,6 +852,22 @@ class PictogramJSONList(ListView): # ############## # +def stats(request): + last_week = date.today() - timedelta(days=7) + return simple_json_response( + **{ + "maps_count": Map.objects.count(), + "maps_active_last_week_count": Map.objects.filter( + modified_at__gt=last_week + ).count(), + "users_count": User.objects.count(), + "users_active_last_week_count": User.objects.filter( + last_login__gt=last_week + ).count(), + } + ) + + def logout(request): do_logout(request) return simple_json_response(redirect="/") From deb0ab09d3493736839fca92f4a7af98d9851a43 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 23 May 2023 11:51:54 -0400 Subject: [PATCH 2/3] Add one hour cache to the stats view --- umap/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umap/urls.py b/umap/urls.py index 0154aaf1..a6bcfab4 100644 --- a/umap/urls.py +++ b/umap/urls.py @@ -153,7 +153,7 @@ urlpatterns += i18n_patterns( re_path(r"^user/(?P.+)/$", views.user_maps, name="user_maps"), re_path(r"", include(i18n_urls)), ) -urlpatterns += (path("stats/", views.stats, name="stats"),) +urlpatterns += (path("stats/", cache_page(60 * 60)(views.stats), name="stats"),) if settings.DEBUG and settings.MEDIA_ROOT: urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) From 6f72df82b74d25d31cc16f19650eb0712cf8c3e7 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 23 May 2023 12:09:10 -0400 Subject: [PATCH 3/3] Improve stats view testing with another user --- umap/tests/conftest.py | 11 +++++++++++ umap/tests/test_views.py | 8 +++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/umap/tests/conftest.py b/umap/tests/conftest.py index e3d237e3..0f3bd6ce 100644 --- a/umap/tests/conftest.py +++ b/umap/tests/conftest.py @@ -2,6 +2,7 @@ import shutil import tempfile import pytest +from django.core.cache import cache from django.core.signing import get_cookie_signer from .base import DataLayerFactory, MapFactory, UserFactory @@ -12,6 +13,7 @@ TMP_ROOT = tempfile.mkdtemp() def pytest_configure(config): from django.conf import settings + settings.MEDIA_ROOT = TMP_ROOT @@ -19,11 +21,20 @@ def pytest_unconfigure(config): shutil.rmtree(TMP_ROOT, ignore_errors=True) +def pytest_runtest_teardown(): + cache.clear() + + @pytest.fixture def user(): return UserFactory(password="123123") +@pytest.fixture +def user2(): + return UserFactory(username="Averell", password="456456") + + @pytest.fixture def licence(): # Should be created by the migrations. diff --git a/umap/tests/test_views.py b/umap/tests/test_views.py index 228d4dd5..d1f025a7 100644 --- a/umap/tests/test_views.py +++ b/umap/tests/test_views.py @@ -1,6 +1,6 @@ import json import socket -from datetime import date +from datetime import date, timedelta import pytest from django.conf import settings @@ -150,13 +150,15 @@ def test_stats_empty(client): @pytest.mark.django_db -def test_stats_basic(client, map, datalayer): +def test_stats_basic(client, map, datalayer, user2): map.owner.last_login = date.today() map.owner.save() + user2.last_login = date.today() - timedelta(days=8) + user2.save() response = client.get(reverse("stats")) assert json.loads(response.content.decode()) == { "maps_active_last_week_count": 1, "maps_count": 1, "users_active_last_week_count": 1, - "users_count": 1, + "users_count": 2, }