Get rid of sesql and upgrade to django 1.6

README.rst

@@ -25,12 +25,6 @@ Install dependencies and project::
     pip install -r requirements.pip
     pip install -e .
 
-Configure stopwords for SeSQL in running this `script <https://bitbucket.org/liberation/sesql/raw/d13763736f3db098127089bce2e66b1be122007a/scripts/generate_stop.sh`>_:
-
-    wget https://bitbucket.org/liberation/sesql/raw/d13763736f3db098127089bce2e66b1be122007a/scripts/generate_stop.sh
-    chmod +x generate_stop.sh
-    sudo generate_stop.sh
-
 Create a default local settings file::
 
     cp umap/settings/local.py.sample umap/settings/local.py

manage.py

@@ -2,14 +2,11 @@
 import os
 import sys
 
-# SeSQL look for "sesql_config" in the sys.path...
-# FIXME: PR to SeSQL to be able to define the import path
-# with an env var
-sys.path.insert(0, os.path.abspath('umap'))
-
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE",
+    os.environ.setdefault(
-        "umap.settings.local")
+        "DJANGO_SETTINGS_MODULE",
+        "umap.settings.local"
+    )
 
     from django.core.management import execute_from_command_line
 

requirements.txt

@@ -1,15 +1,14 @@
 django-compressor==1.3
-Django==1.5.5
+Django==1.6.4
-GenericCache==1.0.2
 httplib2==0.7.7
 oauth2==1.5.211
 oauthlib==0.6.1
-Pillow==2.3.0
+Pillow==2.4.0
 psycopg2==2.5.2
 python-social-auth==0.1.23
 requests-oauthlib==0.4.0
 requests==2.2.1
-simplejson==3.0.7
+simplejson
 South==0.7.6
 django-leaflet-storage==0.5.0
-hg+https://bitbucket.org/liberation/sesql
+django-pgindex==0.8.2

umap/models.py

@@ -0,0 +1,38 @@
+from django.conf import settings
+
+from pgindex import IndexBase, Vector, register
+
+from leaflet_storage.models import Map
+
+
+class UnaccentVector(Vector):
+
+    @property
+    def tsvector(self):
+        if getattr(settings, "UMAP_USE_UNACCENT", False):
+            return u"setweight(to_tsvector('%s', unaccent(E'%s')), '%s')" % (
+                self.dictionary, self.value, self.weight
+            )
+        else:
+            return super(UnaccentVector, self).tsvector
+
+
+class MapIndex(IndexBase):
+
+    def get_title(self):
+        return self.obj.name
+
+    def get_start_publish(self):
+        return self.obj.modified_at
+
+    def get_vectors(self):
+        vectors = []
+        if self.obj.name:
+            vectors.append(UnaccentVector(self.obj.name, weight='A'))
+        if self.obj.description:
+            vectors.append(UnaccentVector(self.obj.description, weight='B'))
+        if self.obj.owner:
+            vectors.append(UnaccentVector(self.obj.owner.username))
+        return vectors
+
+register(Map, MapIndex)

umap/sesql_config.py

@@ -1,148 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) Pilot Systems and Libération, 2010-2011
-
-# This file is part of SeSQL.
-
-# SeSQL is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-
-# Foobar is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with SeSQL.  If not, see <http://www.gnu.org/licenses/>.
-
-#
-# Full text search configuration - we must define that before the imports,
-# because those are used by the imports
-#
-
-# Name of the PostgreSQL Text Search Configuration
-TS_CONFIG_NAME = "simple_french"
-
-# Name of the stopwards file, must be plain ASCII
-STOPWORDS_FILE = "ascii_french"
-
-# Global charset to use
-CHARSET = "utf-8"
-
-from sesql.fields import *
-from sesql.sources import *
-from leaflet_storage import models
-
-#
-# Select the ORM to use
-#
-from sesql.orm.django import DjangoOrmAdapter
-orm = DjangoOrmAdapter()
-
-
-#
-# Fields and tables configuration
-#
-
-# Configuration of SeSQL search fields
-FIELDS = (ClassField("classname"),
-          IntField("id"),
-          DateField("modified_at"),
-          FullTextField("name"),
-          FullTextField("fulltext",
-                        ['name', 'description',
-                          SubField("owner", ["username"])
-                          ],
-                        primary=True,
-                        ),
-          DateField('indexed_at', sql_default='NOW()'),
-          )
-
-# Name of the global lookup table that should contain no real data
-MASTER_TABLE_NAME = "sesql_index"
-
-# Type map, associating Django classes to SeSQL tables
-TYPE_MAP = ((models.Map, "sesql_default"), )
-
-# Additional indexes to create
-CROSS_INDEXES = ()
-
-#
-# Cleanup configuration
-#
-
-from htmlentitydefs import name2codepoint
-from xml.sax import saxutils
-
-html_entities = dict([('&%s;' % k, unichr(v).encode(CHARSET)) for k,v in name2codepoint.items() ])
-ADDITIONAL_CLEANUP_FUNCTION = lambda value: saxutils.unescape(value, html_entities)
-
-#
-# Query configuration
-#
-
-
-# General condition to skip indexing content
-def SKIP_CONDITION(m):
-    return m.share_status != models.Map.PUBLIC
-
-# Default sort order for queries
-DEFAULT_ORDER = ('-modified_at',)
-
-# Default LIMIT in short queries
-DEFAULT_LIMIT = 20
-
-# First we ask for the SMART_QUERY_INITIAL first sorted items
-SMART_QUERY_INITIAL = 2500
-# Then, if we have at least SMART_QUERY_THRESOLD of our limit, we go on
-SMART_QUERY_THRESOLD = 0.35
-# If we have a second query, we do * (wanted/result) * SMART_QUERY_RATIO
-SMART_QUERY_RATIO = 3.0
-
-#
-# Long query cache configuration
-#
-
-# Maximal number of queries to store in the long query cache
-QUERY_CACHE_MAX_SIZE = 10000
-# Life time of a query in the query cache
-QUERY_CACHE_EXPIRY = 24 * 3600
-
-#
-# Daemon configuration
-#
-
-DAEMON_DEFAULT_CHUNK = 100
-DAEMON_DEFAULT_DELAY = 120
-DAEMON_DEFAULT_PID = '/var/run/sesql/update.pid'
-
-#
-# Suggest/history configuration
-#
-
-# default number of hit before including query in db
-HISTORY_DEFAULT_FILTER = 5
-
-# erode factor for time-based decay of recent searches score
-HISTORY_ALPHA = 0.95
-# weight of frequency of the search in the final score
-HISTORY_BETA = 1.0
-# weight of number of results in the final score
-HISTORY_GAMMA = 1.0
-
-# queries to ignore in history
-HISTORY_BLACKLIST = []
-
-
-#
-# Enable sesql searches from Django admin ?
-#
-ENABLE_SESQL_ADMIN = False
-
-#
-# Enable to force all updates to be processed asynchronously by the daemon
-#
-
-ASYNCHRONOUS_INDEXING = False

umap/settings/base.py

@@ -42,7 +42,7 @@ SECRET_KEY = ''
 INSTALLED_APPS = (
     'leaflet_storage',
     'umap',
-    'sesql',
+    'pgindex',
     'compressor',
     'social.apps.django_app.default',
 
@@ -164,6 +164,7 @@ LEAFLET_STORAGE_EXTRA_URLS = {
 SITE_URL = "http://umap.org"
 UMAP_DEMO_SITE = False
 MAP_SHORT_URL_NAME = "umap_short_url"
+UMAP_USE_UNACCENT = False
 
 #==============================================================================
 # Third party app settings

umap/settings/local.py.sample

@@ -66,7 +66,6 @@ LEAFLET_STORAGE_ALLOW_ANONYMOUS = True
 UMAP_DEMO_SITE = True
 SITE_URL = "http://localhost:8019"
 SHORT_SITE_URL = "http://s.hort"
-SESQL_CONFIG_PATH = "umap.sesql_config"
 
 # CACHES = {
 #     'default': {
@@ -77,3 +76,8 @@ SESQL_CONFIG_PATH = "umap.sesql_config"
 
 # POSTGIS_VERSION = (2, 1, 0)
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
+
+# You need to unable accent extension before using UMAP_USE_UNACCENT 
+# python manage.py dbshell
+# CREATE EXTENSION unaccent;
+UMAP_USE_UNACCENT = False

umap/views.py

@@ -15,8 +15,8 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.http import HttpResponse, HttpResponseBadRequest
 from django.utils.translation import ugettext as _
 from django.core.urlresolvers import reverse
-
+from django.db.models.sql.where import ExtraWhere, OR
-from sesql.shortquery import shortquery
+from pgindex import search as pg_search
 
 from leaflet_storage.models import Map
 from leaflet_storage.forms import DEFAULT_CENTER
@@ -124,12 +124,17 @@ class Search(TemplateView, PaginatorMixin):
 
     def get_context_data(self, **kwargs):
         q = self.request.GET.get('q')
-        maps = []
+        results = []
         if q:
-            maps = shortquery(Q(classname='Map') & Q(fulltext__containswords=q))
+            results = pg_search(q)
-            maps = self.paginate(maps)
+            if getattr(settings, 'UMAP_USE_UNACCENT', False):
+                # Add unaccent support
+                results.query.where.add(ExtraWhere(("ts @@ plainto_tsquery('simple', unaccent(%s))", ), [q, ]), OR)
+            results = results.order_by('-rank', '-start_publish')
+            results = self.paginate(results)
+            results.object_list = [Map.objects.get(pk=i.obj_pk) for i in results]
         kwargs.update({
-            'maps': maps,
+            'maps': results,
             'q': q
         })
         return kwargs