Merge pull request #1329 from umap-project/prune-gz

Prune old .gz when saving a datalayer

umap/models.py

@@ -331,6 +331,7 @@ class DataLayer(NamedModel):
             self.geojson.storage.delete(old_name)
             self.geojson.name = new_name
             super(DataLayer, self).save(force_insert, force_update, **kwargs)
+        self.purge_gzip()
         self.purge_old_versions()
 
     def upload_to(self):
@@ -400,13 +401,18 @@ class DataLayer(NamedModel):
         root = self.storage_root()
         names = self.get_versions()[settings.UMAP_KEEP_VERSIONS :]
         for name in names:
-            for ext in ["", ".gz"]:
-                path = os.path.join(root, name + ext)
             try:
-                    self.geojson.storage.delete(path)
+                self.geojson.storage.delete(os.path.join(root, name))
             except FileNotFoundError:
                 pass
 
+    def purge_gzip(self):
+        root = self.storage_root()
+        names = self.geojson.storage.listdir(root)[1]
+        for name in names:
+            if name.startswith(f'{self.pk}_') and name.endswith(".gz"):
+                self.geojson.storage.delete(os.path.join(root, name))
+
 
 class Star(models.Model):
     at = models.DateTimeField(auto_now=True)

umap/tests/test_datalayer.py

@@ -62,20 +62,22 @@ def test_should_remove_old_versions_on_save(datalayer, map, settings):
     settings.UMAP_KEEP_VERSIONS = 3
     root = datalayer.storage_root()
     before = len(datalayer.geojson.storage.listdir(root)[1])
-    newer = '%s/%s_1440924889.geojson' % (root, datalayer.pk)
-    medium = '%s/%s_1440923687.geojson' % (root, datalayer.pk)
-    older = '%s/%s_1440918637.geojson' % (root, datalayer.pk)
-    for path in [medium, newer, older]:
+    newer = f'{root}/{datalayer.pk}_1440924889.geojson'
+    medium = f'{root}/{datalayer.pk}_1440923687.geojson'
+    older = f'{root}/{datalayer.pk}_1440918637.geojson'
+    other = f'{root}/123456_1440918637.geojson'
+    for path in [medium, newer, older, other]:
         datalayer.geojson.storage.save(path, ContentFile("{}"))
         datalayer.geojson.storage.save(path + '.gz', ContentFile("{}"))
-    assert len(datalayer.geojson.storage.listdir(root)[1]) == 6 + before
+    assert len(datalayer.geojson.storage.listdir(root)[1]) == 8 + before
     datalayer.save()
     files = datalayer.geojson.storage.listdir(root)[1]
+    # Flat + gz files, but not latest gz, which is created at first datalayer read.
     assert len(files) == 5
     assert os.path.basename(newer) in files
-    assert os.path.basename(newer + '.gz') in files
     assert os.path.basename(medium) in files
-    assert os.path.basename(medium + '.gz') in files
     assert os.path.basename(datalayer.geojson.path) in files
+    # File from another datalayer, purge should have impacted it.
+    assert os.path.basename(other) in files
+    assert os.path.basename(other + ".gz") in files
     assert os.path.basename(older) not in files
-    assert os.path.basename(older + '.gz') not in files