First shot in Django 1.8 / python 3 support

This also replaced pg_index with a simple builting seach
based on tsvector, as pg_index is not compliant with Django 1.8,
and as some old dependencies that would need to be upgrade too.

README.rst

@@ -21,7 +21,7 @@ Create a virtual environment::
 Install dependencies and project::
 
     cd YOUR_SOURCE_DIR
-    git clone git clone https://bitbucket.org/yohanboniface/umap.git
+    git clone https://bitbucket.org/yohanboniface/umap.git
     pip install -r requirements.txt
     pip install -e .
 
@@ -58,7 +58,7 @@ needs. For example::
     TWITTER_CONSUMER_KEY = "xxx"
     TWITTER_CONSUMER_SECRET = "yyy"
 
-Example of callback URL to use for settings up OAuth apps::
+Example of callback URL to use for setting up OAuth apps::
 
  http://umap.foo.bar/complete/github/
 
@@ -66,7 +66,7 @@ Adapt the `STATIC_ROOT` and `MEDIA_ROOT` to your local environment.
 
 Create the tables::
 
-    python manage.py syncdb --migrate
+    python manage.py migrate
 
 Collect and compress the statics::
 
@@ -87,6 +87,18 @@ Go to the admin (http://localhost:8000/admin/) and add:
 - at least one license
 - at least one tile layer
 
+Search
+------
+
+UMap uses Postgresql tsvector for searching. It case your database is big, you
+may want to add an index. For that, you sould do so:
+
+    CREATE EXTENSION unaccent;
+    CREATE EXTENSION btree_gin;
+    ALTER FUNCTION unaccent(text) IMMUTABLE;
+    ALTER FUNCTION to_tsvector(text) IMMUTABLE;
+    CREATE INDEX search_idx ON leaflet_storage_map USING gin(to_tsvector(unaccent(name)), share_status);
+
 Translating
 -----------
 

requirements.txt

@@ -1,9 +1,7 @@
-django-compressor==1.3
+django-compressor==1.5
-Django==1.6.5
+Django==1.8.3
-Pillow==2.4.0
+Pillow==2.9.0
-psycopg2==2.5.2
+psycopg2==2.6.1
-python-social-auth==0.1.23
+python-social-auth==0.2.12
-simplejson
-South==0.7.6
 django-leaflet-storage==0.7.6
-django-pgindex==0.8.2
+requests==2.7.0

umap/models.py

@@ -1,41 +0,0 @@
-from django.conf import settings
-
-from pgindex import IndexBase, Vector, register
-
-from leaflet_storage.models import Map
-
-
-class UnaccentVector(Vector):
-
-    @property
-    def tsvector(self):
-        if getattr(settings, "UMAP_USE_UNACCENT", False):
-            return u"setweight(to_tsvector('%s', unaccent(E'%s')), '%s')" % (
-                self.dictionary, self.value, self.weight
-            )
-        else:
-            return super(UnaccentVector, self).tsvector
-
-
-class MapIndex(IndexBase):
-
-    def get_title(self):
-        return self.obj.name
-
-    def get_start_publish(self):
-        return self.obj.modified_at
-
-    def get_publish(self):
-        return self.obj.share_status == Map.PUBLIC
-
-    def get_vectors(self):
-        vectors = []
-        if self.obj.name:
-            vectors.append(UnaccentVector(self.obj.name, weight='A'))
-        if self.obj.description:
-            vectors.append(UnaccentVector(self.obj.description, weight='B'))
-        if self.obj.owner:
-            vectors.append(UnaccentVector(self.obj.owner.username))
-        return vectors
-
-register(Map, MapIndex)

umap/settings/base.py

@@ -46,14 +46,6 @@ LANGUAGES = (
 SECRET_KEY = ''
 
 INSTALLED_APPS = (
-    'leaflet_storage',
-    'umap',
-    'pgindex',
-    'compressor',
-    'social.apps.django_app.default',
-
-    'south',
-
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
@@ -61,8 +53,12 @@ INSTALLED_APPS = (
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
-    'django.contrib.admindocs',
+    'django.contrib.gis',
-    'django.contrib.gis'
+
+    'leaflet_storage',
+    'umap',
+    'compressor',
+    'social.apps.django_app.default',
 )
 
 #==============================================================================
@@ -107,7 +103,7 @@ TEMPLATE_DIRS = (
 )
 
 TEMPLATE_CONTEXT_PROCESSORS += (
-    'django.core.context_processors.request',
+    'django.template.context_processors.request',
     'social.apps.django_app.context_processors.backends',
     'social.apps.django_app.context_processors.login_redirect',
     'umap.context_processors.feedback_link',
@@ -139,9 +135,9 @@ MIDDLEWARE_CLASSES = (
 AUTHENTICATION_BACKENDS += (
 )
 
-#==============================================================================
+# =============================================================================
 # Miscellaneous project settings
-#==============================================================================
+# =============================================================================
 LEAFLET_STORAGE_ALLOW_ANONYMOUS = False
 LEAFLET_STORAGE_EXTRA_URLS = {
     'routing': 'http://map.project-osrm.org/?loc={lat},{lng}&hl={locale}',
@@ -151,11 +147,12 @@ SITE_URL = "http://umap.org"
 UMAP_DEMO_SITE = False
 MAP_SHORT_URL_NAME = "umap_short_url"
 UMAP_USE_UNACCENT = False
-UMAP_FEEDBACK_LINK = "http://wiki.openstreetmap.org/wiki/UMap#Feedback_and_help"
+UMAP_FEEDBACK_LINK = "http://wiki.openstreetmap.org/wiki/UMap#Feedback_and_help"  # noqa
+USER_MAPS_URL = 'user_maps'
 
-#==============================================================================
+# =============================================================================
 # Third party app settings
-#==============================================================================
+# =============================================================================
 COMPRESS_ENABLED = True
 COMPRESS_OFFLINE = True
 

umap/templates/leaflet_storage/js.html

@@ -11,12 +11,12 @@
     <script src="{{ STATIC_URL }}storage/reqs/osmtogeojson/osmtogeojson.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/loading/Control.Loading.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/markercluster/leaflet.markercluster-src.js"></script>
-    <script src="{{ STATIC_URL }}storage/reqs/draw/leaflet.draw-src.js"></script>
-    <script src="{{ STATIC_URL }}storage/reqs/measure/leaflet.measurecontrol.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/contextmenu/leaflet.contextmenu-src.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/label/leaflet.label-src.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/georsstogeojson/GeoRSSToGeoJSON.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/heat/leaflet-heat.js"></script>
+    <script src="{{ STATIC_URL }}storage/reqs/toolbar/leaflet.toolbar-src.js"></script>
+    <script src="{{ STATIC_URL }}storage/reqs/formbuilder/Leaflet.FormBuilder.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/togpx/togpx.js"></script>
     <script src="{{ STATIC_URL }}storage/reqs/tokml/tokml.js"></script>
 {% endcompress %}

umap/tests/test_views.py

@@ -9,7 +9,8 @@ from umap.views import validate_url
 
 class TestsValidateProxyURL(TestCase):
 
-    def buildRequest(self, target="http://osm.org/georss.xml", verb="get", **kwargs):
+    def buildRequest(self, target="http://osm.org/georss.xml", verb="get",
+                     **kwargs):
         defaults = {
             'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest',
             'HTTP_REFERER': '%s/path/' % settings.SITE_URL
@@ -72,5 +73,5 @@ class TestsProxy(TestCase):
         }
         response = self.client.get(url, params, **headers)
         self.assertEquals(response.status_code, 200)
-        self.assertIn('Example Domain', response.content)
+        self.assertIn('Example Domain', response.content.decode())
         self.assertNotIn("Cookie", response['Vary'])

umap/urls.py

@@ -17,12 +17,12 @@ urlpatterns = patterns(
     (r'^admin/', include(admin.site.urls)),
     url('', include('social.apps.django_app.urls', namespace='social')),
     url(r'^m/(?P<pk>\d+)/$', MapShortUrl.as_view(), name='umap_short_url'),
-    url(r'^ajax-proxy/$', cache_page(180)(views.ajax_proxy), name='ajax-proxy'),
+    url(r'^ajax-proxy/$', cache_page(180)(views.ajax_proxy), name='ajax-proxy'),  # noqa
 )
 urlpatterns += i18n_patterns(
     '',
     url(r'^$', views.home, name="home"),
-    url(r'^showcase/$', cache_page(24 * 60 * 60)(views.showcase), name='maps_showcase'),
+    url(r'^showcase/$', cache_page(24 * 60 * 60)(views.showcase), name='maps_showcase'),  # noqa
     url(r'^search/$', views.search, name="search"),
     url(r'^about/$', views.about, name="about"),
     url(r'^user/(?P<username>[-_\w@]+)/$', views.user_maps, name='user_maps'),
@@ -30,5 +30,6 @@ urlpatterns += i18n_patterns(
 )
 
 if settings.DEBUG and settings.MEDIA_ROOT:
-    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    urlpatterns += static(settings.MEDIA_URL,
+                          document_root=settings.MEDIA_ROOT)
 urlpatterns += staticfiles_urlpatterns()

umap/views.py

@@ -1,9 +1,15 @@
-import simplejson
+import json
 import mimetypes
-import urllib2
 import socket
 
-from urlparse import urlparse
+try:
+    # python3
+    from urllib.parse import urlparse
+    from urllib.request import Request, build_opener
+    from urllib.error import HTTPError
+except ImportError:
+    from urlparse import urlparse
+    from urllib2 import Request, HTTPError, build_opener
 
 from django.views.generic import TemplateView
 from django.contrib.auth import get_user_model
@@ -15,8 +21,7 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.http import HttpResponse, HttpResponseBadRequest
 from django.utils.translation import ugettext as _
 from django.core.urlresolvers import reverse
-from django.db.models.sql.where import ExtraWhere, OR
+from django.core.validators import URLValidator, ValidationError
-from pgindex import search as pg_search
 
 from leaflet_storage.models import Map
 from leaflet_storage.forms import DEFAULT_CENTER
@@ -36,7 +41,8 @@ class PaginatorMixin(object):
             # If page is not an integer, deliver first page.
             qs = paginator.page(1)
         except EmptyPage:
-            # If page is out of range (e.g. 9999), deliver last page of results.
+            # If page is out of range (e.g. 9999), deliver last page of
+            # results.
             qs = paginator.page(paginator.num_pages)
         return qs
 
@@ -47,7 +53,7 @@ class Home(TemplateView, PaginatorMixin):
 
     def get_context_data(self, **kwargs):
         qs = Map.public
-        if not 'spatialite' in settings.DATABASES['default']['ENGINE']:
+        if 'spatialite' not in settings.DATABASES['default']['ENGINE']:
             # Unsupported query type for sqlite.
             qs = qs.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1)))
         demo_map = None
@@ -103,8 +109,10 @@ class UserMaps(DetailView, PaginatorMixin):
     context_object_name = "current_user"
 
     def get_context_data(self, **kwargs):
-        manager = Map.objects if self.request.user == self.object else Map.public
+        manager = Map.objects if self.request.user == self.object\
-        maps = manager.filter(Q(owner=self.object) | Q(editors=self.object)).distinct().order_by('-modified_at')[:50]
+                  else Map.public
+        maps = manager.filter(Q(owner=self.object) | Q(editors=self.object))
+        maps = maps.distinct().order_by('-modified_at')[:50]
         maps = self.paginate(maps)
         kwargs.update({
             "maps": maps
@@ -131,13 +139,14 @@ class Search(TemplateView, PaginatorMixin):
         q = self.request.GET.get('q')
         results = []
         if q:
-            results = pg_search(q)
+            where = "to_tsvector(name) @@ to_tsquery(%s)"
             if getattr(settings, 'UMAP_USE_UNACCENT', False):
-                # Add unaccent support
+                where = "to_tsvector(unaccent(name)) @@ to_tsquery(unaccent(%s))"  # noqa
-                results.query.where.add(ExtraWhere(("ts @@ plainto_tsquery('simple', unaccent(%s))", ), [q, ]), OR)
+            results = Map.objects.filter(share_status=Map.PUBLIC)
-            results = results.order_by('-rank', '-start_publish')
+            results = results.extra(where=[where], params=[q])
+            results = results.order_by('-modified_at')
+            print(results.query)
             results = self.paginate(results)
-            results.object_list = [Map.objects.get(pk=i.obj_pk) for i in results]
         kwargs.update({
             'maps': results,
             'q': q
@@ -159,7 +168,8 @@ search = Search.as_view()
 class MapsShowCase(View):
 
     def get(self, *args, **kwargs):
-        maps = Map.public.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1))).order_by('-modified_at')[:2500]
+        maps = Map.public.filter(center__distance_gt=(DEFAULT_CENTER, D(km=1)))
+        maps = maps.order_by('-modified_at')[:2500]
 
         def make(m):
             description = m.description or ""
@@ -167,11 +177,13 @@ class MapsShowCase(View):
                 description = u"{description}\n{by} [[{url}|{name}]]".format(
                     description=description,
                     by=_("by"),
-                    url=reverse('user_maps', kwargs={"username": m.owner.username}),
+                    url=reverse('user_maps',
+                                kwargs={"username": m.owner.username}),
                     name=m.owner,
                 )
-            description = u"{}\n[[{}|{}]]".format(description, m.get_absolute_url(), _("View the map"))
+            description = u"{}\n[[{}|{}]]".format(
-            geometry = m.settings['geometry'] if "geometry" in m.settings else simplejson.loads(m.center.geojson)
+                description, m.get_absolute_url(), _("View the map"))
+            geometry = m.settings.get('geometry', json.loads(m.center.geojson))
             return {
                 "type": "Feature",
                 "geometry": geometry,
@@ -185,14 +197,11 @@ class MapsShowCase(View):
             "type": "FeatureCollection",
             "features": [make(m) for m in maps]
         }
-        return HttpResponse(simplejson.dumps(geojson))
+        return HttpResponse(json.dumps(geojson))
 
 showcase = MapsShowCase.as_view()
 
 
-from django.core.validators import URLValidator, ValidationError
-
-
 def validate_url(request):
     assert request.method == "GET"
     assert request.is_ajax()
@@ -226,22 +235,24 @@ class AjaxProxy(View):
         # You should not use this in production (use Nginx or so)
         try:
             url = validate_url(self.request)
-        except AssertionError:
+        except AssertionError as e:
             return HttpResponseBadRequest()
         headers = {
             'User-Agent': 'uMapProxy +http://wiki.openstreetmap.org/wiki/UMap'
         }
-        request = urllib2.Request(url, headers=headers)
+        request = Request(url, headers=headers)
-        opener = urllib2.build_opener()
+        opener = build_opener()
         try:
             proxied_request = opener.open(request)
-        except urllib2.HTTPError as e:
+        except HTTPError as e:
-            return HttpResponse(e.msg, status=e.code, mimetype='text/plain')
+            return HttpResponse(e.msg, status=e.code,
+                                content_type='text/plain')
         else:
             status_code = proxied_request.code
-            mimetype = proxied_request.headers.typeheader or mimetypes.guess_type(url)
+            mimetype = proxied_request.headers.get('Content-Type') or mimetypes.guess_type(url)  # noqa
             content = proxied_request.read()
             # Quick hack to prevent Django from adding a Vary: Cookie header
             self.request.session.accessed = False
-            return HttpResponse(content, status=status_code, mimetype=mimetype)
+            return HttpResponse(content, status=status_code,
+                                content_type=mimetype)
 ajax_proxy = AjaxProxy.as_view()