From 0be42d39cb263f27ef7cdc839c13abd1e9dccff1 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Wed, 8 Nov 2023 15:20:59 -0500 Subject: [PATCH 1/7] Full map download endpoint --- umap/tests/test_map_views.py | 59 ++++++++++++++++++++++++++++++++++++ umap/urls.py | 5 +++ umap/views.py | 26 ++++++++++++++-- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/umap/tests/test_map_views.py b/umap/tests/test_map_views.py index 01e78c59..92ff196f 100644 --- a/umap/tests/test_map_views.py +++ b/umap/tests/test_map_views.py @@ -603,3 +603,62 @@ def test_can_send_link_on_anonymous_map_with_cookie(cookieclient, anonymap): assert resp.status_code == 200 assert len(mail.outbox) == 1 assert mail.outbox[0].subject == "The uMap edit link for your map: test map" + + +def test_download(client, map, datalayer): + url = reverse("map_download", args=(map.pk,)) + response = client.get(url) + assert response.status_code == 200 + # Test response is a json + j = json.loads(response.content.decode()) + assert j["type"] == "umap" + assert j["uri"] == "http://testserver/en/map/test-map_1" + assert j["geometry"] == { + "coordinates": [13.447265624999998, 48.94415123418794], + "type": "Point", + } + assert j["properties"] == { + "datalayersControl": True, + "description": "Which is just the Danube, at the end", + "displayCaptionOnLoad": False, + "displayDataBrowserOnLoad": False, + "displayPopupFooter": False, + "licence": "", + "miniMap": False, + "moreControl": True, + "name": "Cruising on the Donau", + "scaleControl": True, + "tilelayer": { + "attribution": "© OSM Contributors", + "maxZoom": 18, + "minZoom": 0, + "url_template": "http://{s}.osm.fr/{z}/{x}/{y}.png", + }, + "tilelayersControl": True, + "zoom": 7, + "zoomControl": True, + } + assert j["layers"] == [ + { + "_umap_options": { + "browsable": True, + "displayOnLoad": True, + "name": "test datalayer", + }, + "features": [ + { + "geometry": { + "coordinates": [13.68896484375, 48.55297816440071], + "type": "Point", + }, + "properties": { + "_umap_options": {"color": "DarkCyan", "iconClass": "Ball"}, + "description": "Da place anonymous again 755", + "name": "Here", + }, + "type": "Feature", + } + ], + "type": "FeatureCollection", + }, + ] diff --git a/umap/urls.py b/umap/urls.py index 630ea45a..9aeae2ea 100644 --- a/umap/urls.py +++ b/umap/urls.py @@ -39,6 +39,11 @@ urlpatterns = [ ), re_path(r"^i18n/", include("django.conf.urls.i18n")), re_path(r"^agnocomplete/", include("agnocomplete.urls")), + re_path( + r"^map/(?P\d+)/download/", + can_view_map(views.MapDownload.as_view()), + name="map_download", + ), ] i18n_urls = [ diff --git a/umap/views.py b/umap/views.py index 0d341585..c1b2f59a 100644 --- a/umap/views.py +++ b/umap/views.py @@ -469,9 +469,7 @@ class MapDetailMixin: else: map_statuses = AnonymousMapPermissionsForm.STATUS datalayer_statuses = AnonymousDataLayerPermissionsForm.STATUS - properties["edit_statuses"] = [ - (i, str(label)) for i, label in map_statuses - ] + properties["edit_statuses"] = [(i, str(label)) for i, label in map_statuses] properties["datalayer_edit_statuses"] = [ (i, str(label)) for i, label in datalayer_statuses ] @@ -606,6 +604,28 @@ class MapView(MapDetailMixin, PermissionsMixin, DetailView): return Star.objects.filter(by=user, map=self.object).exists() +class MapDownload(DetailView): + model = Map + pk_url_kwarg = "map_id" + + def get_canonical_url(self): + return reverse("map_download", args=(self.object.pk,)) + + def render_to_response(self, context, *args, **kwargs): + geojson = self.object.settings + geojson["type"] = "umap" + geojson["uri"] = self.request.build_absolute_uri(self.object.get_absolute_url()) + datalayers = [] + for datalayer in self.object.datalayer_set.all(): + with open(datalayer.geojson.path, "rb") as f: + layer = json.loads(f.read()) + if datalayer.settings: + layer["_umap_options"] = datalayer.settings + datalayers.append(layer) + geojson["layers"] = datalayers + return simple_json_response(**geojson) + + class MapViewGeoJSON(MapView): def get_canonical_url(self): return reverse("map_geojson", args=(self.object.pk,)) From 207c47d078fa18f2c639d307f3e15393e4a3db66 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Thu, 9 Nov 2023 15:22:10 -0500 Subject: [PATCH 2/7] Button to download umap backup from backend url --- umap/static/umap/js/umap.controls.js | 38 +++++++++++----------------- umap/static/umap/js/umap.js | 20 +-------------- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/umap/static/umap/js/umap.controls.js b/umap/static/umap/js/umap.controls.js index 3898613b..b95494fe 100644 --- a/umap/static/umap/js/umap.controls.js +++ b/umap/static/umap/js/umap.controls.js @@ -924,15 +924,6 @@ L.U.Map.include({ ext: '.csv', filetype: 'text/csv', }, - umap: { - name: L._('Full map data'), - formatter: function (map) { - return map.serialize() - }, - ext: '.umap', - filetype: 'application/json', - selected: true, - }, }, renderEditToolbar: function () { @@ -1140,6 +1131,20 @@ L.U.Map.include({ shortUrl.value = this.options.shortUrl } L.DomUtil.create('hr', '', container) + L.DomUtil.add('h4', '', container, L._('Backup data')) + const downloadUrl = L.Util.template(this.options.urls.map_download, { + map_id: this.options.umap_id, + }) + const link = L.DomUtil.createLink( + 'button', + container, + L._('Download uMap backup format'), + downloadUrl + ) + let name = this.options.name || 'data' + name = name.replace(/[^a-z0-9]/gi, '_').toLowerCase() + link.setAttribute('download', `${name}.umap`) + L.DomUtil.create('hr', '', container) L.DomUtil.add('h4', '', container, L._('Download data')) const typeInput = L.DomUtil.create('select', '', container) typeInput.name = 'format' @@ -1149,12 +1154,6 @@ L.U.Map.include({ container, L._('Only visible features will be downloaded.') ) - exportCaveat.id = 'export_caveat_text' - const toggleCaveat = () => { - if (typeInput.value === 'umap') exportCaveat.style.display = 'none' - else exportCaveat.style.display = 'inherit' - } - L.DomEvent.on(typeInput, 'change', toggleCaveat) for (const key in this.EXPORT_TYPES) { if (this.EXPORT_TYPES.hasOwnProperty(key)) { option = L.DomUtil.create('option', '', typeInput) @@ -1163,18 +1162,11 @@ L.U.Map.include({ if (this.EXPORT_TYPES[key].selected) option.selected = true } } - toggleCaveat() L.DomUtil.createButton( 'button', container, L._('Download data'), - () => { - if (typeInput.value === 'umap') { - this.fullDownload() - } else { - this.download(typeInput.value) - } - }, + () => this.download(typeInput.value), this ) this.ui.openPanel({ data: { html: container } }) diff --git a/umap/static/umap/js/umap.js b/umap/static/umap/js/umap.js index 3e1a9fe3..12de29ca 100644 --- a/umap/static/umap/js/umap.js +++ b/umap/static/umap/js/umap.js @@ -831,7 +831,7 @@ L.U.Map.include({ }, format: function (mode) { - const type = this.EXPORT_TYPES[mode || 'umap'] + const type = this.EXPORT_TYPES[mode || 'TODISCUSS'] const content = type.formatter(this) let name = this.options.name || 'data' name = name.replace(/[^a-z0-9]/gi, '_').toLowerCase() @@ -1074,24 +1074,6 @@ L.U.Map.include({ return properties }, - serialize: function () { - // Do not use local path during unit tests - const uri = window.location.protocol === 'file:' ? null : window.location.href - const umapfile = { - type: 'umap', - uri: uri, - properties: this.exportOptions(), - geometry: this.geometry(), - layers: [], - } - - this.eachDataLayer((datalayer) => { - umapfile.layers.push(datalayer.umapGeoJSON()) - }) - - return JSON.stringify(umapfile, null, 2) - }, - saveSelf: function () { const geojson = { type: 'Feature', From 3a0bcd76da831d88f676cce969211a7560696d3f Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 14 Nov 2023 12:16:47 -0500 Subject: [PATCH 3/7] Align link styles to button ones --- umap/static/umap/content.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/umap/static/umap/content.css b/umap/static/umap/content.css index c354166e..50f3a9cd 100644 --- a/umap/static/umap/content.css +++ b/umap/static/umap/content.css @@ -319,7 +319,8 @@ table.maps thead tr { display: none; } .leaflet-container a.button { - color: #eeeeec; + color: #323737; + font-size: 13.3px; } /* ************************************************* */ From 97fa8c27546e13252348215febabb3d7cb304e70 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 14 Nov 2023 13:26:52 -0500 Subject: [PATCH 4/7] Allow the ?download option in URL --- umap/static/umap/js/umap.js | 14 +++++--------- umap/views.py | 6 +++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/umap/static/umap/js/umap.js b/umap/static/umap/js/umap.js index 12de29ca..ed54f43b 100644 --- a/umap/static/umap/js/umap.js +++ b/umap/static/umap/js/umap.js @@ -272,7 +272,10 @@ L.U.Map.include({ url.searchParams.delete('edit') history.pushState({}, '', url) } - if (L.Util.queryString('download')) this.download() + if (L.Util.queryString('download')) + window.location = L.Util.template(this.options.urls.map_download, { + map_id: this.options.umap_id, + }) }) window.onbeforeunload = () => this.isDirty || null @@ -396,7 +399,6 @@ L.U.Map.include({ }, loadDatalayers: function (force) { - force = force || L.Util.queryString('download') // In case we are in download mode, let's go strait to loading all data const total = this.datalayers_index.length // toload => datalayer metadata remaining to load (synchronous) // dataToload => datalayer data remaining to load (asynchronous) @@ -824,14 +826,8 @@ L.U.Map.include({ }) }, - fullDownload: function () { - // Make sure all data is loaded before downloading - this.once('dataloaded', () => this.download()) - this.loadDatalayers(true) // Force load - }, - format: function (mode) { - const type = this.EXPORT_TYPES[mode || 'TODISCUSS'] + const type = this.EXPORT_TYPES[mode] const content = type.formatter(this) let name = this.options.name || 'data' name = name.replace(/[^a-z0-9]/gi, '_').toLowerCase() diff --git a/umap/views.py b/umap/views.py index c1b2f59a..a7b4dcc1 100644 --- a/umap/views.py +++ b/umap/views.py @@ -623,7 +623,11 @@ class MapDownload(DetailView): layer["_umap_options"] = datalayer.settings datalayers.append(layer) geojson["layers"] = datalayers - return simple_json_response(**geojson) + response = simple_json_response(**geojson) + response[ + "Content-Disposition" + ] = f'attachment; filename="umap_backup_{self.object.slug}.umap"' + return response class MapViewGeoJSON(MapView): From 30e83a143c606993e9159b799ce16cbf957a77cf Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 14 Nov 2023 13:37:02 -0500 Subject: [PATCH 5/7] Tests download view for permissions --- umap/tests/test_map_views.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/umap/tests/test_map_views.py b/umap/tests/test_map_views.py index 92ff196f..52bd98b3 100644 --- a/umap/tests/test_map_views.py +++ b/umap/tests/test_map_views.py @@ -662,3 +662,24 @@ def test_download(client, map, datalayer): "type": "FeatureCollection", }, ] + + +@pytest.mark.parametrize("share_status", [Map.PRIVATE, Map.BLOCKED]) +def test_download_shared_status_map(client, map, datalayer, share_status): + map.share_status = share_status + map.save() + url = reverse("map_download", args=(map.pk,)) + response = client.get(url) + assert response.status_code == 403 + + +def test_download_my_map(client, map, datalayer): + map.share_status = Map.PRIVATE + map.save() + client.login(username=map.owner.username, password="123123") + url = reverse("map_download", args=(map.pk,)) + response = client.get(url) + assert response.status_code == 200 + # Test response is a json + j = json.loads(response.content.decode()) + assert j["type"] == "umap" From 227424366290f865d26d0a7bb6046c70cda76309 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 14 Nov 2023 14:10:25 -0500 Subject: [PATCH 6/7] Fix integration tests for download view --- umap/static/umap/js/umap.controls.js | 2 +- umap/static/umap/test/Map.Export.js | 126 ---------------------- umap/tests/base.py | 5 +- umap/tests/integration/test_export_map.py | 23 +--- umap/tests/test_map_views.py | 6 +- umap/tests/test_views.py | 6 +- 6 files changed, 13 insertions(+), 155 deletions(-) diff --git a/umap/static/umap/js/umap.controls.js b/umap/static/umap/js/umap.controls.js index b95494fe..3eb2ba30 100644 --- a/umap/static/umap/js/umap.controls.js +++ b/umap/static/umap/js/umap.controls.js @@ -1138,7 +1138,7 @@ L.U.Map.include({ const link = L.DomUtil.createLink( 'button', container, - L._('Download uMap backup format'), + L._('Download full data'), downloadUrl ) let name = this.options.name || 'data' diff --git a/umap/static/umap/test/Map.Export.js b/umap/static/umap/test/Map.Export.js index 09ce3588..56630e16 100644 --- a/umap/static/umap/test/Map.Export.js +++ b/umap/static/umap/test/Map.Export.js @@ -101,131 +101,5 @@ describe('L.U.Map.Export', function () { 'name polyname poly11.25,53.585984 10.151367,52.975108 12.689209,52.167194 14.084473,53.199452 12.634277,53.618579 11.25,53.585984 11.25,53.585984test[object Object]test-0.274658,52.57635test[object Object]test-0.571289,54.476422 0.439453,54.610255 1.724854,53.448807 4.163818,53.988395 5.306396,53.533778 6.591797,53.709714 7.042236,53.350551' assert.equal(content, expected) }) - - it('should export to umap', function () { - const { content, filetype, filename } = this.map.format('umap') - assert.equal(filetype, 'application/json') - assert.equal(filename, 'name_of_the_map.umap') - const expected = { - type: 'umap', - uri: null, - properties: { - easing: false, - embedControl: true, - fullscreenControl: true, - searchControl: true, - datalayersControl: true, - zoomControl: true, - permanentCreditBackground: true, - slideshow: {}, - captionMenus: true, - captionBar: false, - limitBounds: {}, - overlay: null, - tilelayer: { - attribution: 'HOT and friends', - name: 'HOT OSM-fr server', - url_template: 'http://{s}.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png', - rank: 99, - minZoom: 0, - maxZoom: 20, - id: 2, - }, - licence: '', - description: 'The description of the map', - name: 'name of the map', - displayPopupFooter: false, - miniMap: false, - moreControl: true, - scaleControl: true, - scrollWheelZoom: true, - zoom: 6, - }, - geometry: { - type: 'Point', - coordinates: [5.0592041015625, 52.05924589011585], - }, - layers: [ - { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - properties: { - name: 'name poly', - }, - geometry: { - type: 'Polygon', - coordinates: [ - [ - [11.25, 53.585984], - [10.151367, 52.975108], - [12.689209, 52.167194], - [14.084473, 53.199452], - [12.634277, 53.618579], - [11.25, 53.585984], - [11.25, 53.585984], - ], - ], - }, - }, - { - type: 'Feature', - properties: { - _umap_options: { - color: 'OliveDrab', - }, - name: 'test', - }, - geometry: { - type: 'Point', - coordinates: [-0.274658, 52.57635], - }, - }, - { - type: 'Feature', - properties: { - _umap_options: { - fill: false, - }, - name: 'test', - }, - geometry: { - type: 'LineString', - coordinates: [ - [-0.571289, 54.476422], - [0.439453, 54.610255], - [1.724854, 53.448807], - [4.163818, 53.988395], - [5.306396, 53.533778], - [6.591797, 53.709714], - [7.042236, 53.350551], - ], - }, - }, - ], - _umap_options: { - displayOnLoad: true, - browsable: true, - editMode: 'advanced', - iconClass: 'Default', - name: 'Elephants', - id: 62, - pictogram_url: null, - opacity: null, - weight: null, - fillColor: '', - color: '', - stroke: true, - smoothFactor: null, - dashArray: '', - fillOpacity: null, - fill: true, - }, - }, - ], - } - assert.deepEqual(JSON.parse(content), expected) - }) }) }) diff --git a/umap/tests/base.py b/umap/tests/base.py index dc8c1717..ddfa21e5 100644 --- a/umap/tests/base.py +++ b/umap/tests/base.py @@ -71,13 +71,11 @@ class MapFactory(factory.django.DjangoModelFactory): "properties": { "datalayersControl": True, "description": "Which is just the Danube, at the end", - "displayCaptionOnLoad": False, - "displayDataBrowserOnLoad": False, "displayPopupFooter": False, "licence": "", "miniMap": False, "moreControl": True, - "name": "Cruising on the Donau", + "name": name, "scaleControl": True, "tilelayer": { "attribution": "\xa9 OSM Contributors", @@ -100,6 +98,7 @@ class MapFactory(factory.django.DjangoModelFactory): def _adjust_kwargs(cls, **kwargs): # Make sure there is no persistency kwargs["settings"] = copy.deepcopy(kwargs["settings"]) + kwargs["settings"]["properties"]["name"] = kwargs["name"] return kwargs class Meta: diff --git a/umap/tests/integration/test_export_map.py b/umap/tests/integration/test_export_map.py index 6a4368dd..a7f7f9d0 100644 --- a/umap/tests/integration/test_export_map.py +++ b/umap/tests/integration/test_export_map.py @@ -9,12 +9,12 @@ pytestmark = pytest.mark.django_db def test_umap_export(map, live_server, datalayer, page): page.goto(f"{live_server.url}{map.get_absolute_url()}?share") - button = page.get_by_role("button", name="Download data") - expect(button).to_be_visible() + link = page.get_by_role("link", name="Download full data") + expect(link).to_be_visible() with page.expect_download() as download_info: - button.click() + link.click() download = download_info.value - assert download.suggested_filename == "test_map.umap" + assert download.suggested_filename == "umap_backup_test-map.umap" path = Path("/tmp/") / download.suggested_filename download.save_as(path) downloaded = json.loads(path.read_text()) @@ -29,14 +29,12 @@ def test_umap_export(map, live_server, datalayer, page): "_umap_options": { "browsable": True, "displayOnLoad": True, - "editMode": "disabled", - "inCaption": True, "name": "test datalayer", }, "features": [ { "geometry": { - "coordinates": [13.688965, 48.552978], + "coordinates": [13.68896484375, 48.55297816440071], "type": "Point", }, "properties": { @@ -51,25 +49,14 @@ def test_umap_export(map, live_server, datalayer, page): } ], "properties": { - "captionBar": False, - "captionMenus": True, "datalayersControl": True, "description": "Which is just the Danube, at the end", "displayPopupFooter": False, - "easing": False, - "embedControl": True, - "fullscreenControl": True, "licence": "", - "limitBounds": {}, "miniMap": False, "moreControl": True, "name": "test map", - "overlay": None, - "permanentCreditBackground": True, "scaleControl": True, - "scrollWheelZoom": True, - "searchControl": True, - "slideshow": {}, "tilelayer": { "attribution": "© OSM Contributors", "maxZoom": 18, diff --git a/umap/tests/test_map_views.py b/umap/tests/test_map_views.py index 52bd98b3..88dda5f0 100644 --- a/umap/tests/test_map_views.py +++ b/umap/tests/test_map_views.py @@ -612,7 +612,7 @@ def test_download(client, map, datalayer): # Test response is a json j = json.loads(response.content.decode()) assert j["type"] == "umap" - assert j["uri"] == "http://testserver/en/map/test-map_1" + assert j["uri"] == f"http://testserver/en/map/test-map_{map.pk}" assert j["geometry"] == { "coordinates": [13.447265624999998, 48.94415123418794], "type": "Point", @@ -620,13 +620,11 @@ def test_download(client, map, datalayer): assert j["properties"] == { "datalayersControl": True, "description": "Which is just the Danube, at the end", - "displayCaptionOnLoad": False, - "displayDataBrowserOnLoad": False, "displayPopupFooter": False, "licence": "", "miniMap": False, "moreControl": True, - "name": "Cruising on the Donau", + "name": "test map", "scaleControl": True, "tilelayer": { "attribution": "© OSM Contributors", diff --git a/umap/tests/test_views.py b/umap/tests/test_views.py index ed11f3b3..08c4586b 100644 --- a/umap/tests/test_views.py +++ b/umap/tests/test_views.py @@ -289,8 +289,8 @@ def test_user_dashboard_display_user_maps(client, map): def test_user_dashboard_display_user_maps_distinct(client, map): # cf https://github.com/umap-project/umap/issues/1325 anonymap = MapFactory(name="Map witout owner should not appear") - user1 = UserFactory(username='user1') - user2 = UserFactory(username='user2') + user1 = UserFactory(username="user1") + user2 = UserFactory(username="user2") map.editors.add(user1) map.editors.add(user2) map.save() @@ -298,7 +298,7 @@ def test_user_dashboard_display_user_maps_distinct(client, map): response = client.get(reverse("user_dashboard")) assert response.status_code == 200 body = response.content.decode() - assert body.count(map.name) == 1 + assert body.count(f'test map') == 1 assert body.count(anonymap.name) == 0 From bf4e481f283645fa80dc899c7a247a65f00da5e8 Mon Sep 17 00:00:00 2001 From: David Larlet Date: Tue, 14 Nov 2023 14:19:54 -0500 Subject: [PATCH 7/7] Fix download umap data JS tests --- umap/static/umap/test/Controls.js | 1 - umap/static/umap/test/_pre.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/umap/static/umap/test/Controls.js b/umap/static/umap/test/Controls.js index 00616316..ff18cf74 100644 --- a/umap/static/umap/test/Controls.js +++ b/umap/static/umap/test/Controls.js @@ -73,7 +73,6 @@ describe('L.U.Controls', function () { happen.click(qs('.umap-browse-actions .umap-browse-link')) assert.equal(qsa('#browse_data_datalayer_62 ul li').length, 3) }) - }) describe('#exportPanel()', function () { diff --git a/umap/static/umap/test/_pre.js b/umap/static/umap/test/_pre.js index 5df2e152..4a5872a7 100644 --- a/umap/static/umap/test/_pre.js +++ b/umap/static/umap/test/_pre.js @@ -132,6 +132,7 @@ function initMap(options) { datalayer_version: '/datalayer/{pk}/{name}', pictogram_list_json: '/pictogram/json/', map_update_permissions: '/map/{map_id}/update/permissions/', + map_download: '/map/{map_id}/download/', }, default_iconUrl: '../src/img/marker.png', zoom: 6,