diff --git a/umap/fields.py b/umap/fields.py index 4798b0b4..4e6506ee 100644 --- a/umap/fields.py +++ b/umap/fields.py @@ -4,6 +4,8 @@ import six from django.db import models from django.utils.encoding import smart_str +from .utils import json_dumps + class DictField(models.TextField): """ @@ -14,7 +16,7 @@ class DictField(models.TextField): if not value: value = {} if not isinstance(value, six.string_types): - value = json.dumps(value) + value = json_dumps(value) return value def from_db_value(self, value, expression, connection): diff --git a/umap/migrations/0018_datalayer_uuid.py b/umap/migrations/0018_datalayer_uuid.py new file mode 100644 index 00000000..86c52ec0 --- /dev/null +++ b/umap/migrations/0018_datalayer_uuid.py @@ -0,0 +1,62 @@ +# Generated by Django 4.2.8 on 2024-02-19 17:40 + +import uuid + +from django.db import migrations, models + +drop_index = """DO $$ +BEGIN + EXECUTE 'ALTER TABLE umap_datalayer DROP CONSTRAINT ' || ( + SELECT indexname + FROM pg_indexes + WHERE tablename = 'umap_datalayer' AND indexname LIKE '%pk%' + ); +END $$; +""" + + +class Migration(migrations.Migration): + dependencies = [ + ("umap", "0017_migrate_to_openstreetmap_oauth2"), + ] + + operations = [ + # Add the new uuid field + migrations.AddField( + model_name="datalayer", + name="uuid", + field=models.UUIDField( + default=uuid.uuid4, editable=False, null=True, serialize=False + ), + ), + # Generate UUIDs for existing records + migrations.RunSQL( + "UPDATE umap_datalayer SET uuid = gen_random_uuid()", + reverse_sql=migrations.RunSQL.noop, + ), + # Remove the primary key constraint + migrations.RunSQL(drop_index, reverse_sql=migrations.RunSQL.noop), + # Drop the "id" primary key… + migrations.AlterField( + "datalayer", name="id", field=models.IntegerField(null=True, blank=True) + ), + # Rename "id" to "old id" + migrations.RenameField( + model_name="datalayer", old_name="id", new_name="old_id" + ), + # … to put it back on the "uuid" + migrations.AlterField( + model_name="datalayer", + name="uuid", + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + primary_key=True, + serialize=False, + ), + ), + # When applying the migration backwards, we need to drop the pk index + # Before addding a new one. + migrations.RunSQL(migrations.RunSQL.noop, reverse_sql=drop_index), + ] diff --git a/umap/migrations/0019_migrate_internal_remote_datalayers.py b/umap/migrations/0019_migrate_internal_remote_datalayers.py new file mode 100644 index 00000000..1a0bc77e --- /dev/null +++ b/umap/migrations/0019_migrate_internal_remote_datalayers.py @@ -0,0 +1,52 @@ +# Generated by Django 4.2 on 2024-02-26 14:09 + +import re + +from django.conf import settings +from django.db import migrations +from django.urls import NoReverseMatch, reverse + +# Some users hacked uMap to use another map datalayer as a remote data source. +# This script gently handles the migration for them. + + +def migrate_datalayers(apps, schema_editor): + DataLayer = apps.get_model("umap", "DataLayer") + + datalayers = DataLayer.objects.filter( + settings__remoteData__url__icontains="datalayer" + ) + + for item in datalayers: + old_url = item.settings["remoteData"]["url"] + match = re.search( + rf"{settings.SITE_URL}/datalayer/(?P\d+)/(?P\d+)", + old_url, + ) + if match: + remote_id = match.group("datalayer_id") + map_id = match.group("map_id") + try: + remote_uuid = DataLayer.objects.get(old_id=remote_id).uuid + except DataLayer.DoesNotExist: + pass + else: + try: + new_url = settings.SITE_URL + reverse( + "datalayer_view", args=[map_id, remote_uuid] + ) + except NoReverseMatch: + pass + else: + item.settings["remoteData"]["url"] = new_url + item.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("umap", "0018_datalayer_uuid"), + ] + + operations = [ + migrations.RunPython(migrate_datalayers, reverse_code=migrations.RunPython.noop) + ] diff --git a/umap/models.py b/umap/models.py index 4bff5f75..aa60b334 100644 --- a/umap/models.py +++ b/umap/models.py @@ -1,6 +1,7 @@ import json import os import time +import uuid from django.conf import settings from django.contrib.auth.models import User @@ -373,7 +374,10 @@ class DataLayer(NamedModel): (EDITORS, _("Editors only")), (OWNER, _("Owner only")), ) - + uuid = models.UUIDField( + unique=True, primary_key=True, default=uuid.uuid4, editable=False + ) + old_id = models.IntegerField(null=True, blank=True) map = models.ForeignKey(Map, on_delete=models.CASCADE) description = models.TextField(blank=True, null=True, verbose_name=_("description")) geojson = models.FileField(upload_to=upload_to, blank=True, null=True) @@ -436,6 +440,7 @@ class DataLayer(NamedModel): def clone(self, map_inst=None): new = self.__class__.objects.get(pk=self.pk) + new._state.adding = True new.pk = None if map_inst: new.map = map_inst @@ -444,7 +449,10 @@ class DataLayer(NamedModel): return new def is_valid_version(self, name): - return name.startswith("%s_" % self.pk) and name.endswith(".geojson") + valid_prefixes = [name.startswith("%s_" % self.pk)] + if self.old_id: + valid_prefixes.append(name.startswith("%s_" % self.old_id)) + return any(valid_prefixes) and name.endswith(".geojson") def version_metadata(self, name): els = name.split(".")[0].split("_") diff --git a/umap/templatetags/umap_tags.py b/umap/templatetags/umap_tags.py index 2349c579..aa7a45c0 100644 --- a/umap/templatetags/umap_tags.py +++ b/umap/templatetags/umap_tags.py @@ -1,9 +1,10 @@ -import json from copy import copy from django import template from django.conf import settings +from umap.utils import json_dumps + register = template.Library() @@ -25,7 +26,7 @@ def map_fragment(map_instance, **kwargs): page = kwargs.pop("page", None) or "" unique_id = prefix + str(page) + "_" + str(map_instance.pk) return { - "map_settings": json.dumps(map_settings), + "map_settings": json_dumps(map_settings), "map": map_instance, "unique_id": unique_id, } diff --git a/umap/tests/integration/test_collaborative_editing.py b/umap/tests/integration/test_collaborative_editing.py index 5d7b80f0..26f6f527 100644 --- a/umap/tests/integration/test_collaborative_editing.py +++ b/umap/tests/integration/test_collaborative_editing.py @@ -98,7 +98,7 @@ def test_collaborative_editing_create_markers(context, live_server, tilelayer): "name": "test datalayer", "inCaption": True, "editMode": "advanced", - "id": datalayer.pk, + "id": str(datalayer.pk), "permissions": {"edit_status": 1}, } @@ -116,7 +116,7 @@ def test_collaborative_editing_create_markers(context, live_server, tilelayer): "name": "test datalayer", "inCaption": True, "editMode": "advanced", - "id": datalayer.pk, + "id": str(datalayer.pk), "permissions": {"edit_status": 1}, } expect(marker_pane_p1).to_have_count(4) @@ -136,7 +136,7 @@ def test_collaborative_editing_create_markers(context, live_server, tilelayer): "name": "test datalayer", "inCaption": True, "editMode": "advanced", - "id": datalayer.pk, + "id": str(datalayer.pk), "permissions": {"edit_status": 1}, } expect(marker_pane_p2).to_have_count(5) diff --git a/umap/tests/test_datalayer_views.py b/umap/tests/test_datalayer_views.py index 3fe8a99d..22ca9b95 100644 --- a/umap/tests/test_datalayer_views.py +++ b/umap/tests/test_datalayer_views.py @@ -111,7 +111,7 @@ def test_update(client, datalayer, map, post_data): # Test response is a json j = json.loads(response.content.decode()) assert "id" in j - assert datalayer.pk == j["id"] + assert str(datalayer.pk) == j["id"] assert j["browsable"] is True assert Path(modified_datalayer.geojson.path).exists() @@ -225,6 +225,41 @@ def test_versions_should_return_versions(client, datalayer, map, settings): assert version in versions["versions"] +def test_versions_can_return_old_format(client, datalayer, map, settings): + map.share_status = Map.PUBLIC + map.save() + root = datalayer.storage_root() + datalayer.old_id = 123 # old datalayer id (now replaced by uuid) + datalayer.save() + + datalayer.geojson.storage.save( + "%s/%s_1440924889.geojson" % (root, datalayer.pk), ContentFile("{}") + ) + datalayer.geojson.storage.save( + "%s/%s_1440923687.geojson" % (root, datalayer.pk), ContentFile("{}") + ) + + # store with the id prefix (rather than the uuid) + old_format_version = "%s_1440918637.geojson" % datalayer.old_id + datalayer.geojson.storage.save( + ("%s/" % root) + old_format_version, ContentFile("{}") + ) + + url = reverse("datalayer_versions", args=(map.pk, datalayer.pk)) + versions = json.loads(client.get(url).content.decode()) + assert len(versions["versions"]) == 4 + version = { + "name": old_format_version, + "size": 2, + "at": "1440918637", + } + assert version in versions["versions"] + + client.get( + reverse("datalayer_version", args=(map.pk, datalayer.pk, old_format_version)) + ) + + def test_version_should_return_one_version_geojson(client, datalayer, map): map.share_status = Map.PUBLIC map.save() diff --git a/umap/urls.py b/umap/urls.py index 72fff437..28564b55 100644 --- a/umap/urls.py +++ b/umap/urls.py @@ -72,18 +72,18 @@ i18n_urls = [ ] i18n_urls += decorated_patterns( [can_view_map, cache_control(must_revalidate=True)], - re_path( - r"^datalayer/(?P\d+)/(?P[\d]+)/$", + path( + "datalayer///", views.DataLayerView.as_view(), name="datalayer_view", ), - re_path( - r"^datalayer/(?P\d+)/(?P[\d]+)/versions/$", + path( + "datalayer///versions/", views.DataLayerVersions.as_view(), name="datalayer_versions", ), - re_path( - r"^datalayer/(?P\d+)/(?P[\d]+)/(?P[_\w]+.geojson)$", + path( + "datalayer///", views.DataLayerVersion.as_view(), name="datalayer_version", ), @@ -145,13 +145,13 @@ map_urls = [ views.DataLayerCreate.as_view(), name="datalayer_create", ), - re_path( - r"^map/(?P[\d]+)/datalayer/delete/(?P\d+)/$", + path( + "map//datalayer/delete//", views.DataLayerDelete.as_view(), name="datalayer_delete", ), - re_path( - r"^map/(?P[\d]+)/datalayer/permissions/(?P\d+)/$", + path( + "map//datalayer/permissions//", views.UpdateDataLayerPermissions.as_view(), name="datalayer_permissions", ), @@ -165,8 +165,8 @@ if settings.DEFAULT_FROM_EMAIL: ) ) datalayer_urls = [ - re_path( - r"^map/(?P[\d]+)/datalayer/update/(?P\d+)/$", + path( + "map//datalayer/update//", views.DataLayerUpdate.as_view(), name="datalayer_update", ), diff --git a/umap/utils.py b/umap/utils.py index 003b9b01..26cf581d 100644 --- a/umap/utils.py +++ b/umap/utils.py @@ -1,7 +1,9 @@ import gzip +import json import os from django.conf import settings +from django.core.serializers.json import DjangoJSONEncoder from django.urls import URLPattern, URLResolver, get_resolver @@ -162,3 +164,8 @@ def merge_features(reference: list, latest: list, incoming: list): merged.append(item) return merged + + +def json_dumps(obj, **kwargs): + """Utility using the Django JSON Encoder when dumping objects""" + return json.dumps(obj, cls=DjangoJSONEncoder, **kwargs) diff --git a/umap/views.py b/umap/views.py index 377432ff..be6a900a 100644 --- a/umap/views.py +++ b/umap/views.py @@ -67,7 +67,14 @@ from .forms import ( UserProfileForm, ) from .models import DataLayer, Licence, Map, Pictogram, Star, TileLayer -from .utils import ConflictError, _urls_for_js, gzip_file, is_ajax, merge_features +from .utils import ( + ConflictError, + _urls_for_js, + gzip_file, + is_ajax, + json_dumps, + merge_features, +) User = get_user_model() @@ -315,7 +322,7 @@ class UserDownload(DetailView, SearchMixin): with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file: for map_ in self.get_maps(): umapjson = map_.generate_umapjson(self.request) - geojson_file = io.StringIO(json.dumps(umapjson)) + geojson_file = io.StringIO(json_dumps(umapjson)) file_name = f"umap_backup_{map_.slug}_{map_.pk}.umap" zip_file.writestr(file_name, geojson_file.getvalue()) @@ -354,7 +361,7 @@ class MapsShowCase(View): } geojson = {"type": "FeatureCollection", "features": [make(m) for m in maps]} - return HttpResponse(smart_bytes(json.dumps(geojson))) + return HttpResponse(smart_bytes(json_dumps(geojson))) showcase = MapsShowCase.as_view() @@ -441,7 +448,7 @@ ajax_proxy = AjaxProxy.as_view() def simple_json_response(**kwargs): - return HttpResponse(json.dumps(kwargs), content_type="application/json") + return HttpResponse(json_dumps(kwargs), content_type="application/json") # ############## # @@ -537,7 +544,7 @@ class MapDetailMixin: geojson["properties"] = {} geojson["properties"].update(properties) geojson["properties"]["datalayers"] = self.get_datalayers() - context["map_settings"] = json.dumps(geojson, indent=settings.DEBUG) + context["map_settings"] = json_dumps(geojson, indent=settings.DEBUG) self.set_preconnect(geojson["properties"], context) return context @@ -1106,7 +1113,7 @@ class DataLayerUpdate(FormLessEditMixin, GZipMixin, UpdateView): # Replace the uploaded file by the merged version. self.request.FILES["geojson"].file = BytesIO( - json.dumps(merged).encode("utf-8") + json_dumps(merged).encode("utf-8") ) # Mark the data to be reloaded by form_valid