Prune old .gz when saving a datalayer

We'll not consume them.

fix #1317
This commit is contained in:
Yohan Boniface 2023-09-21 18:56:41 +02:00
parent 99831dc04d
commit b6b9943264
2 changed files with 22 additions and 14 deletions

View file

@ -331,6 +331,7 @@ class DataLayer(NamedModel):
self.geojson.storage.delete(old_name)
self.geojson.name = new_name
super(DataLayer, self).save(force_insert, force_update, **kwargs)
self.purge_gzip()
self.purge_old_versions()
def upload_to(self):
@ -400,12 +401,17 @@ class DataLayer(NamedModel):
root = self.storage_root()
names = self.get_versions()[settings.UMAP_KEEP_VERSIONS :]
for name in names:
for ext in ["", ".gz"]:
path = os.path.join(root, name + ext)
try:
self.geojson.storage.delete(path)
except FileNotFoundError:
pass
try:
self.geojson.storage.delete(os.path.join(root, name))
except FileNotFoundError:
pass
def purge_gzip(self):
root = self.storage_root()
names = self.geojson.storage.listdir(root)[1]
for name in names:
if name.startswith(f'{self.pk}_') and name.endswith(".gz"):
self.geojson.storage.delete(os.path.join(root, name))
class Star(models.Model):

View file

@ -62,20 +62,22 @@ def test_should_remove_old_versions_on_save(datalayer, map, settings):
settings.UMAP_KEEP_VERSIONS = 3
root = datalayer.storage_root()
before = len(datalayer.geojson.storage.listdir(root)[1])
newer = '%s/%s_1440924889.geojson' % (root, datalayer.pk)
medium = '%s/%s_1440923687.geojson' % (root, datalayer.pk)
older = '%s/%s_1440918637.geojson' % (root, datalayer.pk)
for path in [medium, newer, older]:
newer = f'{root}/{datalayer.pk}_1440924889.geojson'
medium = f'{root}/{datalayer.pk}_1440923687.geojson'
older = f'{root}/{datalayer.pk}_1440918637.geojson'
other = f'{root}/123456_1440918637.geojson'
for path in [medium, newer, older, other]:
datalayer.geojson.storage.save(path, ContentFile("{}"))
datalayer.geojson.storage.save(path + '.gz', ContentFile("{}"))
assert len(datalayer.geojson.storage.listdir(root)[1]) == 6 + before
assert len(datalayer.geojson.storage.listdir(root)[1]) == 8 + before
datalayer.save()
files = datalayer.geojson.storage.listdir(root)[1]
# Flat + gz files, but not latest gz, which is created at first datalayer read.
assert len(files) == 5
assert os.path.basename(newer) in files
assert os.path.basename(newer + '.gz') in files
assert os.path.basename(medium) in files
assert os.path.basename(medium + '.gz') in files
assert os.path.basename(datalayer.geojson.path) in files
# File from another datalayer, purge should have impacted it.
assert os.path.basename(other) in files
assert os.path.basename(other + ".gz") in files
assert os.path.basename(older) not in files
assert os.path.basename(older + '.gz') not in files