jeff/umap
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1329 from umap-project/prune-gz

Browse source 
Prune old .gz when saving a datalayer
This commit is contained in:
Yohan Boniface 2023-09-22 17:14:31 +02:00 committed by GitHub
commit 9bcc18b790
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
umap/models.py
View file

@ -331,6 +331,7 @@ class DataLayer(NamedModel):
self.geojson.storage.delete(old_name) self.geojson.storage.delete(old_name)
self.geojson.name = new_name self.geojson.name = new_name
super(DataLayer, self).save(force_insert, force_update, **kwargs) super(DataLayer, self).save(force_insert, force_update, **kwargs)
self.purge_gzip()
self.purge_old_versions() self.purge_old_versions()
def upload_to(self): def upload_to(self):
@ -400,12 +401,17 @@ class DataLayer(NamedModel):
root = self.storage_root() root = self.storage_root()
names = self.get_versions()[settings.UMAP_KEEP_VERSIONS :] names = self.get_versions()[settings.UMAP_KEEP_VERSIONS :]
for name in names: for name in names:
for ext in ["", ".gz"]: try:
path = os.path.join(root, name + ext) self.geojson.storage.delete(os.path.join(root, name))
try: except FileNotFoundError:
self.geojson.storage.delete(path) pass
except FileNotFoundError:
pass def purge_gzip(self):
root = self.storage_root()
names = self.geojson.storage.listdir(root)[1]
for name in names:
if name.startswith(f'{self.pk}_') and name.endswith(".gz"):
self.geojson.storage.delete(os.path.join(root, name))
class Star(models.Model): class Star(models.Model):

18
umap/tests/test_datalayer.py
View file

@ -62,20 +62,22 @@ def test_should_remove_old_versions_on_save(datalayer, map, settings):
settings.UMAP_KEEP_VERSIONS = 3 settings.UMAP_KEEP_VERSIONS = 3
root = datalayer.storage_root() root = datalayer.storage_root()
before = len(datalayer.geojson.storage.listdir(root)[1]) before = len(datalayer.geojson.storage.listdir(root)[1])
newer = '%s/%s_1440924889.geojson' % (root, datalayer.pk) newer = f'{root}/{datalayer.pk}_1440924889.geojson'
medium = '%s/%s_1440923687.geojson' % (root, datalayer.pk) medium = f'{root}/{datalayer.pk}_1440923687.geojson'
older = '%s/%s_1440918637.geojson' % (root, datalayer.pk) older = f'{root}/{datalayer.pk}_1440918637.geojson'
for path in [medium, newer, older]: other = f'{root}/123456_1440918637.geojson'
for path in [medium, newer, older, other]:
datalayer.geojson.storage.save(path, ContentFile("{}")) datalayer.geojson.storage.save(path, ContentFile("{}"))
datalayer.geojson.storage.save(path + '.gz', ContentFile("{}")) datalayer.geojson.storage.save(path + '.gz', ContentFile("{}"))
assert len(datalayer.geojson.storage.listdir(root)[1]) == 6 + before assert len(datalayer.geojson.storage.listdir(root)[1]) == 8 + before
datalayer.save() datalayer.save()
files = datalayer.geojson.storage.listdir(root)[1] files = datalayer.geojson.storage.listdir(root)[1]
# Flat + gz files, but not latest gz, which is created at first datalayer read.
assert len(files) == 5 assert len(files) == 5
assert os.path.basename(newer) in files assert os.path.basename(newer) in files
assert os.path.basename(newer + '.gz') in files
assert os.path.basename(medium) in files assert os.path.basename(medium) in files
assert os.path.basename(medium + '.gz') in files
assert os.path.basename(datalayer.geojson.path) in files assert os.path.basename(datalayer.geojson.path) in files
# File from another datalayer, purge should have impacted it.
assert os.path.basename(other) in files
assert os.path.basename(other + ".gz") in files
assert os.path.basename(older) not in files assert os.path.basename(older) not in files
assert os.path.basename(older + '.gz') not in files