From 4541578b9ba2b213629131b05794c2408c8b8c85 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 19 Jan 2024 12:27:21 +0100 Subject: [PATCH 1/5] feat: compress static in collectstatic post_process --- pyproject.toml | 2 ++ umap/utils.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3fa6d9e8..0e7c7d34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,8 @@ dependencies = [ "Pillow==10.0.1", "psycopg2==2.9.6", "requests==2.31.0", + "rcssmin==1.1.2", + "rjsmin==1.2.2", "social-auth-core==4.4.2", "social-auth-app-django==5.2.0", ] diff --git a/umap/utils.py b/umap/utils.py index a18f567d..27f973ca 100644 --- a/umap/utils.py +++ b/umap/utils.py @@ -1,9 +1,12 @@ import gzip import os +from pathlib import Path from django.conf import settings from django.contrib.staticfiles.storage import ManifestStaticFilesStorage from django.urls import URLPattern, URLResolver, get_resolver +from rcssmin import cssmin +from rjsmin import jsmin def _urls_for_js(urls=None): @@ -213,3 +216,16 @@ class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage): ), # Remove JS source map rewriting ) + + def post_process(self, paths, **options): + collected = super().post_process(paths, **options) + for original_path, processed_path, processed in collected: + if processed_path.endswith(".js"): + path = Path(settings.STATIC_ROOT) / processed_path + minified = jsmin(path.read_text()) + path.write_text(minified) + if processed_path.endswith(".css"): + path = Path(settings.STATIC_ROOT) / processed_path + minified = cssmin(path.read_text()) + path.write_text(minified) + yield original_path, processed_path, True From df3ed76f3e92f9ea15b79da7e0e6c78d24c21d3c Mon Sep 17 00:00:00 2001 From: David Larlet Date: Fri, 19 Jan 2024 09:22:09 -0500 Subject: [PATCH 2/5] feat: generate/link source map files for JS and CSS --- umap/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/umap/utils.py b/umap/utils.py index 27f973ca..e61a882f 100644 --- a/umap/utils.py +++ b/umap/utils.py @@ -222,10 +222,18 @@ class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage): for original_path, processed_path, processed in collected: if processed_path.endswith(".js"): path = Path(settings.STATIC_ROOT) / processed_path - minified = jsmin(path.read_text()) + initial = path.read_text() + path_map = path.with_suffix(f"{path.suffix}.map") + minified = jsmin(initial) + minified += f"\n//# sourceMappingURL={path_map.name}" path.write_text(minified) + path_map.write_text(initial) if processed_path.endswith(".css"): path = Path(settings.STATIC_ROOT) / processed_path - minified = cssmin(path.read_text()) + initial = path.read_text() + path_map = path.with_suffix(f"{path.suffix}.map") + minified = cssmin(initial) + minified += f"\n//# sourceMappingURL={path_map.name}" path.write_text(minified) + path_map.write_text(initial) yield original_path, processed_path, True From 23af4c60cd1dc3d5445ca2a39ede6955fc9e5de1 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 19 Jan 2024 13:47:46 +0100 Subject: [PATCH 3/5] chore: move UmapManifestStaticFilesStorage to a dedicated file --- umap/settings/base.py | 2 +- umap/storage.py | 77 ++++++++++++++++++++++++++ umap/tests/integration/test_statics.py | 2 +- umap/utils.py | 75 ------------------------- 4 files changed, 79 insertions(+), 77 deletions(-) create mode 100644 umap/storage.py diff --git a/umap/settings/base.py b/umap/settings/base.py index b890d7fc..08f3030d 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -169,7 +169,7 @@ STORAGES = { "BACKEND": "django.core.files.storage.FileSystemStorage", }, "staticfiles": { - "BACKEND": "umap.utils.UmapManifestStaticFilesStorage", + "BACKEND": "umap.storage.UmapManifestStaticFilesStorage", }, } diff --git a/umap/storage.py b/umap/storage.py new file mode 100644 index 00000000..a09bb871 --- /dev/null +++ b/umap/storage.py @@ -0,0 +1,77 @@ +from pathlib import Path + +from django.conf import settings +from django.contrib.staticfiles.storage import ManifestStaticFilesStorage +from rcssmin import cssmin +from rjsmin import jsmin + + +class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage): + support_js_module_import_aggregation = True + + # We remove `;` at the end of all regexps to match our prettier config. + _js_module_import_aggregation_patterns = ( + "*.js", + ( + ( + ( + r"""(?Pimport(?s:(?P[\s\{].*?))""" + r"""\s*from\s*['"](?P[\.\/].*?)["']\s*)""" + ), + 'import%(import)s from "%(url)s"\n', + ), + ( + ( + r"""(?Pexport(?s:(?P[\s\{].*?))""" + r"""\s*from\s*["'](?P[\.\/].*?)["']\s*)""" + ), + 'export%(exports)s from "%(url)s"\n', + ), + ( + r"""(?Pimport\s*['"](?P[\.\/].*?)["']\s*)""", + 'import"%(url)s"\n', + ), + ( + r"""(?Pimport\(["'](?P.*?)["']\))""", + """import("%(url)s")""", + ), + ), + ) + + # https://github.com/django/django/blob/0fcee1676c7f14bb08e2cc662898dee56d9cf207↩ + # /django/contrib/staticfiles/storage.py#L78C5-L105C6 + patterns = ( + ( + "*.css", + ( + r"""(?Purl\(['"]{0,1}\s*(?P.*?)["']{0,1}\))""", + ( + r"""(?P@import\s*["']\s*(?P.*?)["'])""", + """@import url("%(url)s")""", + ), + # Remove CSS source map rewriting + ), + ), + # Remove JS source map rewriting + ) + + def post_process(self, paths, **options): + collected = super().post_process(paths, **options) + for original_path, processed_path, processed in collected: + if processed_path.endswith(".js"): + path = Path(settings.STATIC_ROOT) / processed_path + initial = path.read_text() + path_map = path.with_suffix(f"{path.suffix}.map") + minified = jsmin(initial) + minified += f"\n//# sourceMappingURL={path_map.name}" + path.write_text(minified) + path_map.write_text(initial) + if processed_path.endswith(".css"): + path = Path(settings.STATIC_ROOT) / processed_path + initial = path.read_text() + path_map = path.with_suffix(f"{path.suffix}.map") + minified = cssmin(initial) + minified += f"\n//# sourceMappingURL={path_map.name}" + path.write_text(minified) + path_map.write_text(initial) + yield original_path, processed_path, True diff --git a/umap/tests/integration/test_statics.py b/umap/tests/integration/test_statics.py index 657609b1..3f833425 100644 --- a/umap/tests/integration/test_statics.py +++ b/umap/tests/integration/test_statics.py @@ -24,7 +24,7 @@ def test_javascript_have_been_loaded( ): settings.STORAGES["staticfiles"][ "BACKEND" - ] = "umap.utils.UmapManifestStaticFilesStorage" + ] = "umap.storage.UmapManifestStaticFilesStorage" datalayer.settings["displayOnLoad"] = False datalayer.save() map.settings["properties"]["defaultView"] = "latest" diff --git a/umap/utils.py b/umap/utils.py index e61a882f..2a51fe4e 100644 --- a/umap/utils.py +++ b/umap/utils.py @@ -1,12 +1,8 @@ import gzip import os -from pathlib import Path from django.conf import settings -from django.contrib.staticfiles.storage import ManifestStaticFilesStorage from django.urls import URLPattern, URLResolver, get_resolver -from rcssmin import cssmin -from rjsmin import jsmin def _urls_for_js(urls=None): @@ -166,74 +162,3 @@ def merge_features(reference: list, latest: list, incoming: list): merged.append(item) return merged - - -class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage): - support_js_module_import_aggregation = True - - # We remove `;` at the end of all regexps to match our prettier config. - _js_module_import_aggregation_patterns = ( - "*.js", - ( - ( - ( - r"""(?Pimport(?s:(?P[\s\{].*?))""" - r"""\s*from\s*['"](?P[\.\/].*?)["']\s*)""" - ), - 'import%(import)s from "%(url)s"\n', - ), - ( - ( - r"""(?Pexport(?s:(?P[\s\{].*?))""" - r"""\s*from\s*["'](?P[\.\/].*?)["']\s*)""" - ), - 'export%(exports)s from "%(url)s"\n', - ), - ( - r"""(?Pimport\s*['"](?P[\.\/].*?)["']\s*)""", - 'import"%(url)s"\n', - ), - ( - r"""(?Pimport\(["'](?P.*?)["']\))""", - """import("%(url)s")""", - ), - ), - ) - - # https://github.com/django/django/blob/0fcee1676c7f14bb08e2cc662898dee56d9cf207↩ - # /django/contrib/staticfiles/storage.py#L78C5-L105C6 - patterns = ( - ( - "*.css", - ( - r"""(?Purl\(['"]{0,1}\s*(?P.*?)["']{0,1}\))""", - ( - r"""(?P@import\s*["']\s*(?P.*?)["'])""", - """@import url("%(url)s")""", - ), - # Remove CSS source map rewriting - ), - ), - # Remove JS source map rewriting - ) - - def post_process(self, paths, **options): - collected = super().post_process(paths, **options) - for original_path, processed_path, processed in collected: - if processed_path.endswith(".js"): - path = Path(settings.STATIC_ROOT) / processed_path - initial = path.read_text() - path_map = path.with_suffix(f"{path.suffix}.map") - minified = jsmin(initial) - minified += f"\n//# sourceMappingURL={path_map.name}" - path.write_text(minified) - path_map.write_text(initial) - if processed_path.endswith(".css"): - path = Path(settings.STATIC_ROOT) / processed_path - initial = path.read_text() - path_map = path.with_suffix(f"{path.suffix}.map") - minified = cssmin(initial) - minified += f"\n//# sourceMappingURL={path_map.name}" - path.write_text(minified) - path_map.write_text(initial) - yield original_path, processed_path, True From 541fbd4e56d087e357774c297fb0e5a79f745b10 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 19 Jan 2024 13:50:26 +0100 Subject: [PATCH 4/5] docs: remove mentions of compress / django-compressor --- docs/config/settings.md | 15 --------------- docs/install.md | 4 +--- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/docs/config/settings.md b/docs/config/settings.md index f9995078..9875c960 100644 --- a/docs/config/settings.md +++ b/docs/config/settings.md @@ -24,21 +24,6 @@ The hosts that uMap expects. Can be set through env var too: `ALLOWED_HOSTS=umap.mydomain.org,u.mydomain.org` -#### COMPRESS_ENABLED -#### COMPRESS_STORAGE - -To activate the compression of the static files, you can set this flag to `True`. - -You can then run the following command to compress the assets: - -```bash -umap compress -``` - -Optionally add `COMPRESS_STORAGE = "compressor.storage.GzipCompressorFileStorage"` -and add `gzip_static on` directive to Nginx `/static` location, so Nginx will -serve pregenerated files instead of compressing them on the fly. - #### DEBUG Set it to `True` for easier debugging in case of error. diff --git a/docs/install.md b/docs/install.md index 77e36a27..8ffc20c6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -143,9 +143,8 @@ Here are the commands you'll need to run to create the tables, collect the stati # Create the database tables umap migrate -# Collect and compress static files +# Collect static files umap collectstatic -umap compress # Create a super user umap createsuperuser @@ -167,5 +166,4 @@ Usually, for upgrading, you need those steps: pip install umap-project --upgrade umap migrate umap collectstatic -umap compress ``` From 283ad4451567c0cd59f3705d194a8ccf0a7e3a5e Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 19 Jan 2024 17:43:18 +0100 Subject: [PATCH 5/5] chore: do not try to create source maps for now It's more complex than what we thought. --- umap/storage.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/umap/storage.py b/umap/storage.py index a09bb871..f3987ba8 100644 --- a/umap/storage.py +++ b/umap/storage.py @@ -61,17 +61,11 @@ class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage): if processed_path.endswith(".js"): path = Path(settings.STATIC_ROOT) / processed_path initial = path.read_text() - path_map = path.with_suffix(f"{path.suffix}.map") minified = jsmin(initial) - minified += f"\n//# sourceMappingURL={path_map.name}" path.write_text(minified) - path_map.write_text(initial) if processed_path.endswith(".css"): path = Path(settings.STATIC_ROOT) / processed_path initial = path.read_text() - path_map = path.with_suffix(f"{path.suffix}.map") minified = cssmin(initial) - minified += f"\n//# sourceMappingURL={path_map.name}" path.write_text(minified) - path_map.write_text(initial) yield original_path, processed_path, True