From 7b5ea8df67132550de33c42e18b74b9a8092fc86 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Sun, 2 Aug 2015 18:08:31 +0200 Subject: [PATCH] First shot in Django 1.8 / python 3 support This also replaced pg_index with a simple builting seach based on tsvector, as pg_index is not compliant with Django 1.8, and as some old dependencies that would need to be upgrade too. --- README.rst | 18 +++++-- requirements.txt | 14 +++--- umap/models.py | 41 --------------- umap/settings/base.py | 29 +++++------ umap/templates/leaflet_storage/js.html | 4 +- umap/tests/test_views.py | 5 +- umap/urls.py | 7 +-- umap/views.py | 69 +++++++++++++++----------- 8 files changed, 83 insertions(+), 104 deletions(-) delete mode 100644 umap/models.py diff --git a/README.rst b/README.rst index c67b3609..912b3445 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Create a virtual environment:: Install dependencies and project:: cd YOUR_SOURCE_DIR - git clone git clone https://bitbucket.org/yohanboniface/umap.git + git clone https://bitbucket.org/yohanboniface/umap.git pip install -r requirements.txt pip install -e . @@ -58,7 +58,7 @@ needs. For example:: TWITTER_CONSUMER_KEY = "xxx" TWITTER_CONSUMER_SECRET = "yyy" -Example of callback URL to use for settings up OAuth apps:: +Example of callback URL to use for setting up OAuth apps:: http://umap.foo.bar/complete/github/ @@ -66,7 +66,7 @@ Adapt the `STATIC_ROOT` and `MEDIA_ROOT` to your local environment. Create the tables:: - python manage.py syncdb --migrate + python manage.py migrate Collect and compress the statics:: @@ -87,6 +87,18 @@ Go to the admin (http://localhost:8000/admin/) and add: - at least one license - at least one tile layer +Search +------ + +UMap uses Postgresql tsvector for searching. It case your database is big, you +may want to add an index. For that, you sould do so: + + CREATE EXTENSION unaccent; + CREATE EXTENSION btree_gin; + ALTER FUNCTION unaccent(text) IMMUTABLE; + ALTER FUNCTION to_tsvector(text) IMMUTABLE; + CREATE INDEX search_idx ON leaflet_storage_map USING gin(to_tsvector(unaccent(name)), share_status); + Translating ----------- diff --git a/requirements.txt b/requirements.txt index 62eda11f..31366227 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,7 @@ -django-compressor==1.3 -Django==1.6.5 -Pillow==2.4.0 -psycopg2==2.5.2 -python-social-auth==0.1.23 -simplejson -South==0.7.6 +django-compressor==1.5 +Django==1.8.3 +Pillow==2.9.0 +psycopg2==2.6.1 +python-social-auth==0.2.12 django-leaflet-storage==0.7.6 -django-pgindex==0.8.2 +requests==2.7.0 diff --git a/umap/models.py b/umap/models.py deleted file mode 100644 index 57db3b20..00000000 --- a/umap/models.py +++ /dev/null @@ -1,41 +0,0 @@ -from django.conf import settings - -from pgindex import IndexBase, Vector, register - -from leaflet_storage.models import Map - - -class UnaccentVector(Vector): - - @property - def tsvector(self): - if getattr(settings, "UMAP_USE_UNACCENT", False): - return u"setweight(to_tsvector('%s', unaccent(E'%s')), '%s')" % ( - self.dictionary, self.value, self.weight - ) - else: - return super(UnaccentVector, self).tsvector - - -class MapIndex(IndexBase): - - def get_title(self): - return self.obj.name - - def get_start_publish(self): - return self.obj.modified_at - - def get_publish(self): - return self.obj.share_status == Map.PUBLIC - - def get_vectors(self): - vectors = [] - if self.obj.name: - vectors.append(UnaccentVector(self.obj.name, weight='A')) - if self.obj.description: - vectors.append(UnaccentVector(self.obj.description, weight='B')) - if self.obj.owner: - vectors.append(UnaccentVector(self.obj.owner.username)) - return vectors - -register(Map, MapIndex) diff --git a/umap/settings/base.py b/umap/settings/base.py index 7e94274f..315d7834 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -46,14 +46,6 @@ LANGUAGES = ( SECRET_KEY = '' INSTALLED_APPS = ( - 'leaflet_storage', - 'umap', - 'pgindex', - 'compressor', - 'social.apps.django_app.default', - - 'south', - 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', @@ -61,8 +53,12 @@ INSTALLED_APPS = ( 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.gis' + 'django.contrib.gis', + + 'leaflet_storage', + 'umap', + 'compressor', + 'social.apps.django_app.default', ) #============================================================================== @@ -107,7 +103,7 @@ TEMPLATE_DIRS = ( ) TEMPLATE_CONTEXT_PROCESSORS += ( - 'django.core.context_processors.request', + 'django.template.context_processors.request', 'social.apps.django_app.context_processors.backends', 'social.apps.django_app.context_processors.login_redirect', 'umap.context_processors.feedback_link', @@ -139,9 +135,9 @@ MIDDLEWARE_CLASSES = ( AUTHENTICATION_BACKENDS += ( ) -#============================================================================== +# ============================================================================= # Miscellaneous project settings -#============================================================================== +# ============================================================================= LEAFLET_STORAGE_ALLOW_ANONYMOUS = False LEAFLET_STORAGE_EXTRA_URLS = { 'routing': 'http://map.project-osrm.org/?loc={lat},{lng}&hl={locale}', @@ -151,11 +147,12 @@ SITE_URL = "http://umap.org" UMAP_DEMO_SITE = False MAP_SHORT_URL_NAME = "umap_short_url" UMAP_USE_UNACCENT = False -UMAP_FEEDBACK_LINK = "http://wiki.openstreetmap.org/wiki/UMap#Feedback_and_help" +UMAP_FEEDBACK_LINK = "http://wiki.openstreetmap.org/wiki/UMap#Feedback_and_help" # noqa +USER_MAPS_URL = 'user_maps' -#============================================================================== +# ============================================================================= # Third party app settings -#============================================================================== +# ============================================================================= COMPRESS_ENABLED = True COMPRESS_OFFLINE = True diff --git a/umap/templates/leaflet_storage/js.html b/umap/templates/leaflet_storage/js.html index 7cc11751..b6bd72db 100644 --- a/umap/templates/leaflet_storage/js.html +++ b/umap/templates/leaflet_storage/js.html @@ -11,12 +11,12 @@ - - + + {% endcompress %} diff --git a/umap/tests/test_views.py b/umap/tests/test_views.py index fb7f00bd..2c73ccba 100644 --- a/umap/tests/test_views.py +++ b/umap/tests/test_views.py @@ -9,7 +9,8 @@ from umap.views import validate_url class TestsValidateProxyURL(TestCase): - def buildRequest(self, target="http://osm.org/georss.xml", verb="get", **kwargs): + def buildRequest(self, target="http://osm.org/georss.xml", verb="get", + **kwargs): defaults = { 'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest', 'HTTP_REFERER': '%s/path/' % settings.SITE_URL @@ -72,5 +73,5 @@ class TestsProxy(TestCase): } response = self.client.get(url, params, **headers) self.assertEquals(response.status_code, 200) - self.assertIn('Example Domain', response.content) + self.assertIn('Example Domain', response.content.decode()) self.assertNotIn("Cookie", response['Vary']) diff --git a/umap/urls.py b/umap/urls.py index 85f171fc..66a3702f 100644 --- a/umap/urls.py +++ b/umap/urls.py @@ -17,12 +17,12 @@ urlpatterns = patterns( (r'^admin/', include(admin.site.urls)), url('', include('social.apps.django_app.urls', namespace='social')), url(r'^m/(?P\d+)/$', MapShortUrl.as_view(), name='umap_short_url'), - url(r'^ajax-proxy/$', cache_page(180)(views.ajax_proxy), name='ajax-proxy'), + url(r'^ajax-proxy/$', cache_page(180)(views.ajax_proxy), name='ajax-proxy'), # noqa ) urlpatterns += i18n_patterns( '', url(r'^$', views.home, name="home"), - url(r'^showcase/$', cache_page(24 * 60 * 60)(views.showcase), name='maps_showcase'), + url(r'^showcase/$', cache_page(24 * 60 * 60)(views.showcase), name='maps_showcase'), # noqa url(r'^search/$', views.search, name="search"), url(r'^about/$', views.about, name="about"), url(r'^user/(?P[-_\w@]+)/$', views.user_maps, name='user_maps'), @@ -30,5 +30,6 @@ urlpatterns += i18n_patterns( ) if settings.DEBUG and settings.MEDIA_ROOT: - urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + urlpatterns += static(settings.MEDIA_URL, + document_root=settings.MEDIA_ROOT) urlpatterns += staticfiles_urlpatterns() diff --git a/umap/views.py b/umap/views.py index 2a7b33a3..87c337b8 100644 --- a/umap/views.py +++ b/umap/views.py @@ -1,9 +1,15 @@ -import simplejson +import json import mimetypes -import urllib2 import socket -from urlparse import urlparse +try: + # python3 + from urllib.parse import urlparse + from urllib.request import Request, build_opener + from urllib.error import HTTPError +except ImportError: + from urlparse import urlparse + from urllib2 import Request, HTTPError, build_opener from django.views.generic import TemplateView from django.contrib.auth import get_user_model @@ -15,8 +21,7 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponse, HttpResponseBadRequest from django.utils.translation import ugettext as _ from django.core.urlresolvers import reverse -from django.db.models.sql.where import ExtraWhere, OR -from pgindex import search as pg_search +from django.core.validators import URLValidator, ValidationError from leaflet_storage.models import Map from leaflet_storage.forms import DEFAULT_CENTER @@ -36,7 +41,8 @@ class PaginatorMixin(object): # If page is not an integer, deliver first page. qs = paginator.page(1) except EmptyPage: - # If page is out of range (e.g. 9999), deliver last page of results. + # If page is out of range (e.g. 9999), deliver last page of + # results. qs = paginator.page(paginator.num_pages) return qs @@ -47,7 +53,7 @@ class Home(TemplateView, PaginatorMixin): def get_context_data(self, **kwargs): qs = Map.public - if not 'spatialite' in settings.DATABASES['default']['ENGINE']: + if 'spatialite' not in settings.DATABASES['default']['ENGINE']: # Unsupported query type for sqlite. qs = qs.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1))) demo_map = None @@ -103,8 +109,10 @@ class UserMaps(DetailView, PaginatorMixin): context_object_name = "current_user" def get_context_data(self, **kwargs): - manager = Map.objects if self.request.user == self.object else Map.public - maps = manager.filter(Q(owner=self.object) | Q(editors=self.object)).distinct().order_by('-modified_at')[:50] + manager = Map.objects if self.request.user == self.object\ + else Map.public + maps = manager.filter(Q(owner=self.object) | Q(editors=self.object)) + maps = maps.distinct().order_by('-modified_at')[:50] maps = self.paginate(maps) kwargs.update({ "maps": maps @@ -131,13 +139,14 @@ class Search(TemplateView, PaginatorMixin): q = self.request.GET.get('q') results = [] if q: - results = pg_search(q) + where = "to_tsvector(name) @@ to_tsquery(%s)" if getattr(settings, 'UMAP_USE_UNACCENT', False): - # Add unaccent support - results.query.where.add(ExtraWhere(("ts @@ plainto_tsquery('simple', unaccent(%s))", ), [q, ]), OR) - results = results.order_by('-rank', '-start_publish') + where = "to_tsvector(unaccent(name)) @@ to_tsquery(unaccent(%s))" # noqa + results = Map.objects.filter(share_status=Map.PUBLIC) + results = results.extra(where=[where], params=[q]) + results = results.order_by('-modified_at') + print(results.query) results = self.paginate(results) - results.object_list = [Map.objects.get(pk=i.obj_pk) for i in results] kwargs.update({ 'maps': results, 'q': q @@ -159,7 +168,8 @@ search = Search.as_view() class MapsShowCase(View): def get(self, *args, **kwargs): - maps = Map.public.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1))).order_by('-modified_at')[:2500] + maps = Map.public.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1))) + maps = maps.order_by('-modified_at')[:2500] def make(m): description = m.description or "" @@ -167,11 +177,13 @@ class MapsShowCase(View): description = u"{description}\n{by} [[{url}|{name}]]".format( description=description, by=_("by"), - url=reverse('user_maps', kwargs={"username": m.owner.username}), + url=reverse('user_maps', + kwargs={"username": m.owner.username}), name=m.owner, ) - description = u"{}\n[[{}|{}]]".format(description, m.get_absolute_url(), _("View the map")) - geometry = m.settings['geometry'] if "geometry" in m.settings else simplejson.loads(m.center.geojson) + description = u"{}\n[[{}|{}]]".format( + description, m.get_absolute_url(), _("View the map")) + geometry = m.settings.get('geometry', json.loads(m.center.geojson)) return { "type": "Feature", "geometry": geometry, @@ -185,14 +197,11 @@ class MapsShowCase(View): "type": "FeatureCollection", "features": [make(m) for m in maps] } - return HttpResponse(simplejson.dumps(geojson)) + return HttpResponse(json.dumps(geojson)) showcase = MapsShowCase.as_view() -from django.core.validators import URLValidator, ValidationError - - def validate_url(request): assert request.method == "GET" assert request.is_ajax() @@ -226,22 +235,24 @@ class AjaxProxy(View): # You should not use this in production (use Nginx or so) try: url = validate_url(self.request) - except AssertionError: + except AssertionError as e: return HttpResponseBadRequest() headers = { 'User-Agent': 'uMapProxy +http://wiki.openstreetmap.org/wiki/UMap' } - request = urllib2.Request(url, headers=headers) - opener = urllib2.build_opener() + request = Request(url, headers=headers) + opener = build_opener() try: proxied_request = opener.open(request) - except urllib2.HTTPError as e: - return HttpResponse(e.msg, status=e.code, mimetype='text/plain') + except HTTPError as e: + return HttpResponse(e.msg, status=e.code, + content_type='text/plain') else: status_code = proxied_request.code - mimetype = proxied_request.headers.typeheader or mimetypes.guess_type(url) + mimetype = proxied_request.headers.get('Content-Type') or mimetypes.guess_type(url) # noqa content = proxied_request.read() # Quick hack to prevent Django from adding a Vary: Cookie header self.request.session.accessed = False - return HttpResponse(content, status=status_code, mimetype=mimetype) + return HttpResponse(content, status=status_code, + content_type=mimetype) ajax_proxy = AjaxProxy.as_view()