Merge branch 'release/0.10.8'

Fix sampling for Dynamic segments

.travis.yml

@@ -4,6 +4,9 @@ language: python
 
 matrix:
   include:
+    - python: 2.7
+      env: lint
+
     - python: 2.7
       env: TOXENV=py27-django111-wagtail113
 
@@ -25,4 +28,4 @@ deploy:
     secure: IxPnc95OFNQsl7kFfLlLc38KfHh/W79VXnhEkdb2x1GZznOsG167QlhpAnyXIJysCQxgMMwLMuPOOdk1WIxOpGNM1/M80hNzPAfxMYWPuwposDdwoIc8SyREPJt16BXWAY+rAH8SHxld9p1YdRbOEPSSglODe4dCmQWsCbKjV3aKv+gZxBvf6pMxUglp2fBIlCwrE77MyMYh9iW0AGzD6atC2xN9NgAm4f+2WFlKCUL/MztLhNWdvWEiibQav6Tts7p8tWrsBVzywDW2IOy3O0ihPgRdISZ7QrxhiJTjlHYPAy4eRGOnYSQXvp6Dy8ONE5a6Uv5g3Xw2UmQo85sSMUs2VxR0G7d+PQgL0A7ENBQ5i7eSAFHYs8EswiGilW2A7mM4qRXwg9URLelYSdkM+aNXvR+25dCsXakiO4NjCz/BPgNzxPeQLlBdxR5vHugeM/XYuhy6CHlZrR/uueaO0X8RyhJcqeDjBy58IrwYS3Mpj7QCfBpQ9PqsqXEWV9BKwKiBXM2+3hkhawFDWa0GW2PDbORKtSLy/ORfGLx5Y9qxQYKEGvFQA3iqkTjrLj+KeUziKtuvEAcvsdBIJVIxeoHwdl+qqxEm8A7YuRBnWVxWc3jE6wBXboeFP92gVe/ueoXmY22riK9Ja0pli3TyNga8by9WM8Qp4D2ZqkVXHwg=
   on:
     tags: true
-    all_branches: true
+    condition: $TOXENV = py27-django111-wagtail113

CHANGES

@@ -1,3 +1,25 @@
+0.10.8
+==================
+ - Don't add users to exclude list for dynamic segments
+ - Store segments a user is excluded from in the session
+
+0.10.7
+==================
+ - Bug Fix: Ensure static segment members are show the survey immediately
+ - Records users excluded by randomisation on the segment
+ - Don't re-check excluded users
+
+0.10.6
+==================
+ - Accepts and stores randomisation percentage for segment
+ - Adds users to segment based on random number relative to percentage
+
+0.10.5
+==================
+ - Count how many users match a segments rules before saving the segment
+ - Stores count on the segment and displays in the dashboard
+ - Enables testing users against rules if there isn't an active request
+
 0.10.0
 ==================
  - Adds static and dynamic segments

docs/conf.py

@@ -55,10 +55,10 @@ author = 'Lab Digital BV'
 # built documents.
 #
 # The short X.Y version.
-version = '0.10.3'
+version = '0.10.8'
 
 # The full version, including alpha/beta/rc tags.
-release = '0.10.3'
+release = '0.10.8'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.3
+current_version = 0.10.8
 commit = true
 tag = true
 tag_name = {new_version}

setup.py

@@ -32,7 +32,7 @@ with open('README.rst') as fh:
 
 setup(
     name='wagtail-personalisation-molo',
-    version='0.10.3',
+    version='0.10.8',
     description='A forked version of Wagtail add-on for showing personalized content',
     author='Praekelt.org',
     author_email='dev@praekeltfoundation.org',

src/wagtail_personalisation/adapters.py

@@ -3,7 +3,6 @@ from __future__ import absolute_import, unicode_literals
 from django.conf import settings
 from django.db.models import F
 from django.utils.module_loading import import_string
-from django.utils import timezone
 
 from wagtail_personalisation.models import Segment
 from wagtail_personalisation.rules import AbstractBaseRule
@@ -67,17 +66,21 @@ class SessionSegmentsAdapter(BaseSegmentsAdapter):
         self.request.session.setdefault('segments', [])
         self._segment_cache = None
 
-    def get_segments(self):
+    def get_segments(self, key="segments"):
         """Return the persistent segments stored in the request session.
 
+        :param key: The key under which the segments are stored
+        :type key: String
         :returns: The segments in the request session
         :rtype: list of wagtail_personalisation.models.Segment or empty list
 
         """
-        if self._segment_cache is not None:
+        if key == "segments" and self._segment_cache is not None:
             return self._segment_cache
 
-        raw_segments = self.request.session['segments']
+        if key not in self.request.session:
+            return []
+        raw_segments = self.request.session[key]
         segment_ids = [segment['id'] for segment in raw_segments]
 
         segments = (
@@ -87,14 +90,17 @@ class SessionSegmentsAdapter(BaseSegmentsAdapter):
             .in_bulk(segment_ids))
 
         retval = [segments[pk] for pk in segment_ids if pk in segments]
+        if key == "segments":
             self._segment_cache = retval
         return retval
 
-    def set_segments(self, segments):
+    def set_segments(self, segments, key="segments"):
         """Set the currently active segments
 
         :param segments: The segments to set for the current request
         :type segments: list of wagtail_personalisation.models.Segment
+        :param key: The key under which to store the segments. Optional
+        :type key: String
 
         """
         cache_segments = []
@@ -109,7 +115,8 @@ class SessionSegmentsAdapter(BaseSegmentsAdapter):
             serialized_segments.append(serialized)
             segment_ids.add(segment.pk)
 
-        self.request.session['segments'] = serialized_segments
+        self.request.session[key] = serialized_segments
+        if key == "segments":
             self._segment_cache = cache_segments
 
     def get_segment_by_id(self, segment_id):
@@ -172,12 +179,16 @@ class SessionSegmentsAdapter(BaseSegmentsAdapter):
         rule_models = AbstractBaseRule.get_descendant_models()
 
         current_segments = self.get_segments()
+        excluded_segments = self.get_segments("excluded_segments")
 
         # Run tests on all remaining enabled segments to verify applicability.
         additional_segments = []
         for segment in enabled_segments:
             if segment.is_static and segment.static_users.filter(id=self.request.user.id).exists():
                 additional_segments.append(segment)
+            elif (segment.excluded_users.filter(id=self.request.user.id).exists() or
+                    segment in excluded_segments):
+                continue
             elif not segment.is_static or not segment.is_full:
                 segment_rules = []
                 for rule_model in rule_models:
@@ -186,14 +197,19 @@ class SessionSegmentsAdapter(BaseSegmentsAdapter):
                 result = self._test_rules(segment_rules, self.request,
                                           match_any=segment.match_any)
 
-                if result and segment.is_static and not segment.is_full:
+                if result and segment.randomise_into_segment():
+                    if segment.is_static and not segment.is_full:
                         if self.request.user.is_authenticated():
                             segment.static_users.add(self.request.user)
-
-                if result:
                     additional_segments.append(segment)
+                elif result:
+                    if segment.is_static and self.request.user.is_authenticated():
+                        segment.excluded_users.add(self.request.user)
+                    else:
+                        excluded_segments += [segment]
 
         self.set_segments(current_segments + additional_segments)
+        self.set_segments(excluded_segments, "excluded_segments")
         self.update_visit_count()
 
 

src/wagtail_personalisation/forms.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
+from datetime import datetime
 from importlib import import_module
 from itertools import takewhile
 
@@ -8,15 +9,11 @@ from django.contrib.auth import get_user_model
 from django.contrib.auth.models import AnonymousUser
 from django.contrib.sessions.models import Session
 from django.contrib.staticfiles.templatetags.staticfiles import static
-from django.core.exceptions import ValidationError
 from django.test.client import RequestFactory
-from django.utils import timezone
 from django.utils.lru_cache import lru_cache
 from django.utils.translation import ugettext_lazy as _
 from wagtail.wagtailadmin.forms import WagtailAdminModelForm
 
-
-
 SessionStore = import_module(settings.SESSION_ENGINE).SessionStore
 
 
@@ -29,8 +26,30 @@ def user_from_data(user_id):
         return AnonymousUser()
 
 
-
 class SegmentAdminForm(WagtailAdminModelForm):
+
+    def count_matching_users(self, rules, match_any):
+        """ Calculates how many users match the given static rules
+        """
+        count = 0
+
+        static_rules = [rule for rule in rules if rule.static]
+
+        if not static_rules:
+            return count
+
+        User = get_user_model()
+        users = User.objects.filter(is_active=True, is_staff=False)
+
+        for user in users.iterator():
+            if match_any:
+                if any(rule.test_user(None, user) for rule in static_rules):
+                    count += 1
+            elif all(rule.test_user(None, user) for rule in static_rules):
+                count += 1
+
+        return count
+
     def clean(self):
         cleaned_data = super(SegmentAdminForm, self).clean()
         Segment = self._meta.model
@@ -62,13 +81,22 @@ class SegmentAdminForm(WagtailAdminModelForm):
             if field not in excluded:
                 form.add_error(field, _('Cannot update a static segment'))
 
-
     def save(self, *args, **kwargs):
         is_new = not self.instance.id
 
         if not self.instance.is_static:
             self.instance.count = 0
 
+        if is_new:
+            rules = [
+                form.instance for formset in self.formsets.values()
+                for form in formset
+                if form not in formset.deleted_forms
+            ]
+            self.instance.matched_users_count = self.count_matching_users(
+                rules, self.instance.match_any)
+            self.instance.matched_count_updated_at = datetime.now()
+
         instance = super(SegmentAdminForm, self).save(*args, **kwargs)
 
         if is_new and instance.is_static and instance.all_rules_static:
@@ -79,6 +107,7 @@ class SegmentAdminForm(WagtailAdminModelForm):
             adapter = get_segment_adapter(request)
 
             users_to_add = []
+            users_to_exclude = []
             sessions = Session.objects.iterator()
             take_session = takewhile(
                 lambda x: instance.count == 0 or len(users_to_add) <= instance.count,
@@ -91,10 +120,13 @@ class SegmentAdminForm(WagtailAdminModelForm):
                     request.user = user
                     request.session = SessionStore(session_key=session.session_key)
                     passes = adapter._test_rules(instance.get_rules(), request, instance.match_any)
-                    if passes:
+                    if passes and instance.randomise_into_segment():
                         users_to_add.append(user)
+                    elif passes:
+                        users_to_exclude.append(user)
 
             instance.static_users.add(*users_to_add)
+            instance.excluded_users.add(*users_to_exclude)
 
         return instance
 

src/wagtail_personalisation/migrations/0011_personalisablepagemetadata.py

@@ -2,8 +2,8 @@
 # Generated by Django 1.11.1 on 2017-05-31 14:28
 from __future__ import unicode_literals
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

src/wagtail_personalisation/migrations/0016_auto_20180125_0918.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.9 on 2018-01-25 09:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wagtail_personalisation', '0015_static_users'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='segment',
+            name='matched_count_updated_at',
+            field=models.DateTimeField(editable=False, null=True),
+        ),
+        migrations.AddField(
+            model_name='segment',
+            name='matched_users_count',
+            field=models.PositiveIntegerField(default=0, editable=False),
+        ),
+    ]

src/wagtail_personalisation/migrations/0017_segment_randomisation_percent.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.8 on 2018-01-31 16:12
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wagtail_personalisation', '0016_auto_20180125_0918'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='segment',
+            name='randomisation_percent',
+            field=models.PositiveSmallIntegerField(blank=True, default=None, help_text='If this number is set each user matching the rules will have this percentage chance of being placed in the segment.', null=True, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(0)]),
+        ),
+    ]

src/wagtail_personalisation/migrations/0018_segment_excluded_users.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.9 on 2018-02-09 08:28
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('wagtail_personalisation', '0017_segment_randomisation_percent'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='segment',
+            name='excluded_users',
+            field=models.ManyToManyField(help_text='Users that matched the rules but were excluded from the segment for some reason e.g. randomisation', related_name='excluded_segments', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

src/wagtail_personalisation/models.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import, unicode_literals
+import random
 
 from django import forms
 from django.conf import settings
-from django.contrib.sessions.models import Session
+from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models, transaction
 from django.template.defaultfilters import slugify
 from django.utils.encoding import python_2_unicode_compatible
@@ -11,11 +12,7 @@ from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 from modelcluster.models import ClusterableModel
 from wagtail.wagtailadmin.edit_handlers import (
-    FieldPanel,
+    FieldPanel, FieldRowPanel, InlinePanel, MultiFieldPanel)
-    FieldRowPanel,
-    InlinePanel,
-    MultiFieldPanel,
-)
 from wagtail.wagtailcore.models import Page
 
 from wagtail_personalisation.rules import AbstractBaseRule
@@ -86,6 +83,25 @@ class Segment(ClusterableModel):
     static_users = models.ManyToManyField(
         settings.AUTH_USER_MODEL,
     )
+    excluded_users = models.ManyToManyField(
+        settings.AUTH_USER_MODEL,
+        help_text=_("Users that matched the rules but were excluded from the "
+                    "segment for some reason e.g. randomisation"),
+        related_name="excluded_segments"
+    )
+
+    matched_users_count = models.PositiveIntegerField(default=0, editable=False)
+    matched_count_updated_at = models.DateTimeField(null=True, editable=False)
+
+    randomisation_percent = models.PositiveSmallIntegerField(
+        null=True, blank=True, default=None,
+        help_text=_(
+            "If this number is set each user matching the rules will "
+            "have this percentage chance of being placed in the segment."
+        ), validators=[
+            MaxValueValidator(100),
+            MinValueValidator(0)
+        ])
 
     objects = SegmentQuerySet.as_manager()
 
@@ -102,6 +118,7 @@ class Segment(ClusterableModel):
                 FieldPanel('match_any'),
                 FieldPanel('type', widget=forms.RadioSelect),
                 FieldPanel('count', classname='count_field'),
+                FieldPanel('randomisation_percent', classname='percent_field'),
             ], heading="Segment"),
             MultiFieldPanel([
                 InlinePanel(
@@ -172,6 +189,19 @@ class Segment(ClusterableModel):
         if save:
             self.save()
 
+    def randomise_into_segment(self):
+        """ Returns True if randomisation_percent is not set or it generates
+        a random number less than the randomisation_percent
+        This is so there is some randomisation in which users are added to the
+        segment
+        """
+        if self.randomisation_percent is None:
+            return True
+
+        if random.randint(1, 100) <= self.randomisation_percent:
+            return True
+        return False
+
 
 class PersonalisablePageMetadata(ClusterableModel):
     """The personalisable page model. Allows creation of variants with linked

src/wagtail_personalisation/rules.py

@@ -220,7 +220,12 @@ class VisitCountRule(AbstractBaseRule):
     class Meta:
         verbose_name = _('Visit count Rule')
 
-    def test_user(self, request):
+    def test_user(self, request, user=None):
+        if user:
+            # This rule currently does not support testing a user directly
+            # TODO: Make this test a user directly when the rule uses
+            # historical data
+            return False
         operator = self.operator
         segment_count = self.count
 
@@ -261,6 +266,7 @@ class QueryRule(AbstractBaseRule):
 
     """
     icon = 'fa-link'
+    static = True
 
     parameter = models.SlugField(_("The query parameter to search for"),
                                  max_length=20)
@@ -275,7 +281,13 @@ class QueryRule(AbstractBaseRule):
     class Meta:
         verbose_name = _('Query Rule')
 
-    def test_user(self, request):
+    def test_user(self, request, user=None):
+        if user:
+            # This rule currently does not support testing a user directly
+            # TODO: Make this test a user directly if/when the rule uses
+            # historical data
+            return False
+
         return request.GET.get(self.parameter, '') == self.value
 
     def description(self):

src/wagtail_personalisation/templates/modeladmin/wagtail_personalisation/segment/dashboard.html

@@ -70,6 +70,13 @@
                                                     {% endif %}
                                                 </li>
 
+                                                {% if segment.randomisation_percent is not None %}
+                                                    <li class="stat_card">
+                                                        <span>{{ segment.randomisation_percent }} %</span>
+                                                        {% trans "Chance that visitors matching the rules are added to the segment" %}
+                                                    </li>
+                                                {% endif %}
+
                                                 {% for rule in segment.get_rules %}
                                                     <li class="stat_card {{ rule.encoded_name }}">
                                                         {{ rule.description.title }}
@@ -80,6 +87,11 @@
                                                         {% endif %}
                                                     </li>
                                                 {% endfor %}
+                                                {% if segment.matched_users_count > 0 %}
+                                                    <li class="stat_card">
+                                                        <span class="icon icon-fa-user"> {{ segment.matched_users_count }} {% trans "user" %}{{ segment.matched_users_count|pluralize }}</span>  {% trans "were possible matches for this segment at creation" %}
+                                                    </li>
+                                                {% endif %}
                                             </ul>
                                         </div>
 

src/wagtail_personalisation/utils.py

@@ -103,9 +103,17 @@ def exclude_variants(pages):
     :return: List of pages that aren't variants
     :rtype: list
     """
-    return [page for page in pages
+    return [
-            if (hasattr(page, 'personalisation_metadata') is False)
+        page for page in pages
-            or (hasattr(page, 'personalisation_metadata')
+        if (
-                and page.personalisation_metadata is None)
+            (
-            or (hasattr(page, 'personalisation_metadata')
+                hasattr(page, 'personalisation_metadata') is False
-                and page.personalisation_metadata.is_canonical)]
+            ) or
+            (
+                hasattr(page, 'personalisation_metadata') and page.personalisation_metadata is None
+            ) or
+            (
+                hasattr(page, 'personalisation_metadata') and page.personalisation_metadata.is_canonical
+            )
+        )
+    ]

src/wagtail_personalisation/wagtail_hooks.py

@@ -7,7 +7,7 @@ from django.core.urlresolvers import reverse
 from django.template.defaultfilters import pluralize
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
-from wagtail.wagtailadmin.site_summary import SummaryItem, PagesSummaryItem
+from wagtail.wagtailadmin.site_summary import PagesSummaryItem, SummaryItem
 from wagtail.wagtailadmin.widgets import Button, ButtonWithDropdownFromHook
 from wagtail.wagtailcore import hooks
 from wagtail.wagtailcore.models import Page

tests/settings.py

@@ -1,10 +1,9 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
-from pkg_resources import parse_version as V
 
 import django
-
+from pkg_resources import parse_version as V
 
 DATABASES = {
     'default': {
@@ -56,6 +55,7 @@ TEMPLATES = [
     },
 ]
 
+
 def get_middleware_settings():
     return (
         'django.middleware.common.CommonMiddleware',
@@ -69,6 +69,7 @@ def get_middleware_settings():
         'wagtail.wagtailcore.middleware.SiteMiddleware',
     )
 
+
 # Django 1.10 started to use "MIDDLEWARE" instead of "MIDDLEWARE_CLASSES".
 if V(django.get_version()) < V('1.10'):
     MIDDLEWARE_CLASSES = get_middleware_settings()

tests/site/pages/migrations/0001_initial.py

@@ -21,7 +21,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='ContentPage',
             fields=[
-                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')),
+                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')),  # noqa: E501
                 ('subtitle', models.CharField(blank=True, default='', max_length=255)),
                 ('body', wagtail.wagtailcore.fields.RichTextField(blank=True, default='')),
             ],

tests/site/pages/migrations/0002_regularpage.py

@@ -2,9 +2,9 @@
 # Generated by Django 1.11.1 on 2017-06-02 04:26
 from __future__ import unicode_literals
 
-from django.db import migrations, models
 import django.db.models.deletion
 import wagtail.wagtailcore.fields
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='RegularPage',
             fields=[
-                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')),
+                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')),  # noqa: E501
                 ('subtitle', models.CharField(blank=True, default='', max_length=255)),
                 ('body', wagtail.wagtailcore.fields.RichTextField(blank=True, default='')),
             ],

tests/unit/test_factories.py

@@ -4,16 +4,14 @@ import datetime
 
 import pytest
 
-from tests.factories.page import ContentPageFactory
+from tests.factories.rule import ReferralRuleFactory, QueryRuleFactory
-from tests.factories.rule import (
-    DayRuleFactory, DeviceRuleFactory, ReferralRuleFactory, TimeRuleFactory)
 from tests.factories.segment import SegmentFactory
-from tests.factories.site import SiteFactory
 from wagtail_personalisation.models import Segment
 from wagtail_personalisation.rules import TimeRule
 
 # Factory tests
 
+
 @pytest.mark.django_db
 def test_segment_create():
     factoried_segment = SegmentFactory()
@@ -27,8 +25,6 @@ def test_segment_create():
     assert factoried_segment.status == segment.status
 
 
-
-
 @pytest.mark.django_db
 def test_referral_rule_create():
     segment = SegmentFactory(name='Referral')
@@ -37,3 +33,16 @@ def test_referral_rule_create():
         segment=segment)
 
     assert referral_rule.regex_string == 'test.test'
+
+
+@pytest.mark.django_db
+def test_query_rule_create():
+    segment = SegmentFactory(name='Query')
+    query_rule = QueryRuleFactory(
+        parameter="query",
+        value="value",
+        segment=segment)
+
+    assert query_rule.parameter == 'query'
+    assert query_rule.value == 'value'
+    assert query_rule.static

tests/unit/test_rules_time.py

@@ -16,6 +16,8 @@ def test_time_rule_create():
         segment=segment)
 
     assert time_rule.start_time == datetime.time(8, 0, 0)
+
+
 @pytest.mark.django_db
 @freeze_time("10:00:00")
 def test_requesttime_segment(client, site):

tests/unit/test_static_dynamic_segments.py

@@ -2,18 +2,18 @@ from __future__ import absolute_import, unicode_literals
 
 import datetime
 
-from django.forms.models import model_to_dict
-from django.contrib.sessions.backends.db import SessionStore
 import pytest
-from wagtail_personalisation.forms import SegmentAdminForm
+from django.forms.models import model_to_dict
-from wagtail_personalisation.models import Segment
-from wagtail_personalisation.rules import TimeRule, VisitCountRule
 
 from tests.factories.segment import SegmentFactory
+from wagtail_personalisation.forms import SegmentAdminForm
+from wagtail_personalisation.models import Segment
+from wagtail_personalisation.rules import (AbstractBaseRule, TimeRule,
+                                           VisitCountRule)
 
 
 def form_with_data(segment, *rules):
-    model_fields = ['type', 'status', 'count', 'name', 'match_any']
+    model_fields = ['type', 'status', 'count', 'name', 'match_any', 'randomisation_percent']
 
     class TestSegmentAdminForm(SegmentAdminForm):
         class Meta:
@@ -63,6 +63,7 @@ def test_anonymous_user_not_added_to_static_segment_at_creation(site, client):
 
     assert not instance.static_users.all()
 
+
 @pytest.mark.django_db
 def test_match_any_correct_populates(site, client, django_user_model):
     user = django_user_model.objects.create(username='first')
@@ -246,3 +247,393 @@ def test_dynamic_segment_with_non_static_rules_have_a_count():
     )
     form = form_with_data(segment, rule)
     assert form.is_valid(), form.errors
+
+
+@pytest.mark.django_db
+def test_randomisation_percentage_added_to_segment_at_creation(site, client, mocker, django_user_model):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    segment.randomisation_percent = 80
+    rule = VisitCountRule()
+
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    assert instance.randomisation_percent == 80
+
+
+@pytest.mark.django_db
+def test_randomisation_percentage_min_zero(site, client, mocker, django_user_model):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    segment.randomisation_percent = -1
+    rule = VisitCountRule()
+
+    form = form_with_data(segment, rule)
+    assert not form.is_valid()
+
+
+@pytest.mark.django_db
+def test_randomisation_percentage_max_100(site, client, mocker, django_user_model):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    segment.randomisation_percent = 101
+    rule = VisitCountRule()
+
+    form = form_with_data(segment, rule)
+    assert not form.is_valid()
+
+
+@pytest.mark.django_db
+def test_in_static_segment_if_random_is_below_percentage(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, count=1,
+                                   randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    mocker.patch('random.randint', return_value=39)
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert instance.id == client.session['segments'][0]['id']
+    assert user in instance.static_users.all()
+    assert user not in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_not_in_static_segment_if_random_is_above_percentage(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, count=1,
+                                   randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    mocker.patch('random.randint', return_value=41)
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert len(client.session['segments']) == 0
+    assert user not in instance.static_users.all()
+    assert user in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_offered_dynamic_segment_if_random_is_below_percentage(site, client, mocker):
+    segment = SegmentFactory.build(type=Segment.TYPE_DYNAMIC,
+                                   randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    mocker.patch('random.randint', return_value=39)
+    session = client.session
+    session.save()
+    client.get(site.root_page.url)
+
+    assert len(client.session['excluded_segments']) == 0
+    assert instance.id == client.session['segments'][0]['id']
+
+
+@pytest.mark.django_db
+def test_not_offered_dynamic_segment_if_random_is_above_percentage(site, client, mocker):
+    segment = SegmentFactory.build(type=Segment.TYPE_DYNAMIC,
+                                   randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    mocker.patch('random.randint', return_value=41)
+    session = client.session
+    session.save()
+    client.get(site.root_page.url)
+
+    assert len(client.session['segments']) == 0
+    assert instance.id == client.session['excluded_segments'][0]['id']
+
+
+@pytest.mark.django_db
+def test_not_in_segment_if_percentage_is_0(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, count=1,
+                                   randomisation_percent=0)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert len(client.session['segments']) == 0
+    assert user not in instance.static_users.all()
+    assert user in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_always_in_segment_if_percentage_is_100(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, count=1,
+                                   randomisation_percent=100)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert instance.id == client.session['segments'][0]['id']
+    assert user in instance.static_users.all()
+    assert user not in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_not_added_to_static_segment_at_creation_if_random_above_percent(site, client, mocker, user):
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    mocker.patch('random.randint', return_value=41)
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    assert user not in instance.static_users.all()
+    assert user in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_added_to_static_segment_at_creation_if_random_below_percent(site, client, mocker, user):
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    mocker.patch('random.randint', return_value=39)
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, randomisation_percent=40)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+
+    assert user in instance.static_users.all()
+    assert user not in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_rules_check_skipped_if_user_in_excluded(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC, count=1,
+                                   randomisation_percent=100)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+    instance.excluded_users.add(user)
+    instance.save
+
+    mock_test_rule = mocker.patch(
+        'wagtail_personalisation.adapters.SessionSegmentsAdapter._test_rules')
+
+    session = client.session
+    session.save()
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert mock_test_rule.call_count == 0
+    assert len(client.session['segments']) == 0
+    assert user not in instance.static_users.all()
+    assert user in instance.excluded_users.all()
+
+
+@pytest.mark.django_db
+def test_rules_check_skipped_if_dynamic_segment_in_excluded(site, client, mocker, user):
+    segment = SegmentFactory.build(type=Segment.TYPE_DYNAMIC,
+                                   randomisation_percent=100)
+    rule = VisitCountRule(counted_page=site.root_page)
+    form = form_with_data(segment, rule)
+    instance = form.save()
+    instance.persistent = True
+    instance.save()
+
+    session = client.session
+    session['excluded_segments'] = [{'id': instance.pk}]
+    session.save()
+
+    mock_test_rule = mocker.patch(
+        'wagtail_personalisation.adapters.SessionSegmentsAdapter._test_rules')
+
+    client.force_login(user)
+    client.get(site.root_page.url)
+
+    assert mock_test_rule.call_count == 0
+    assert len(client.session['segments']) == 0
+
+
+@pytest.mark.django_db
+def test_matched_user_count_added_to_segment_at_creation(site, client, mocker, django_user_model):
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second')
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    rule = VisitCountRule()
+
+    form = form_with_data(segment, rule)
+    mock_test_user = mocker.patch('wagtail_personalisation.rules.VisitCountRule.test_user', return_value=True)
+    instance = form.save()
+
+    assert mock_test_user.call_count == 2
+    instance.matched_users_count = 2
+
+
+@pytest.mark.django_db
+def test_count_users_matching_static_rules(site, client, django_user_model):
+    class TestStaticRule(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            return True
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second')
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    rule = TestStaticRule()
+    form = form_with_data(segment, rule)
+
+    assert form.count_matching_users([rule], True) is 2
+
+
+@pytest.mark.django_db
+def test_count_matching_users_excludes_staff(site, client, django_user_model):
+    class TestStaticRule(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            return True
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second', is_staff=True)
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    rule = TestStaticRule()
+    form = form_with_data(segment, rule)
+
+    assert form.count_matching_users([rule], True) is 1
+
+
+@pytest.mark.django_db
+def test_count_matching_users_excludes_inactive(site, client, django_user_model):
+    class TestStaticRule(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            return True
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second', is_active=False)
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    rule = TestStaticRule()
+    form = form_with_data(segment, rule)
+
+    assert form.count_matching_users([rule], True) is 1
+
+
+@pytest.mark.django_db
+def test_count_matching_users_only_counts_static_rules(site, client, django_user_model):
+    class TestStaticRule(AbstractBaseRule):
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            return True
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second')
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    rule = TestStaticRule()
+    form = form_with_data(segment, rule)
+
+    assert form.count_matching_users([rule], True) is 0
+
+
+@pytest.mark.django_db
+def test_count_matching_users_handles_match_any(site, client, django_user_model):
+    class TestStaticRuleFirst(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            if user.username == 'first':
+                return True
+            return False
+
+    class TestStaticRuleSecond(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            if user.username == 'second':
+                return True
+            return False
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second')
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    first_rule = TestStaticRuleFirst()
+    second_rule = TestStaticRuleSecond()
+    form = form_with_data(segment, first_rule, second_rule)
+
+    assert form.count_matching_users([first_rule, second_rule], True) is 2
+
+
+@pytest.mark.django_db
+def test_count_matching_users_handles_match_all(site, client, django_user_model):
+    class TestStaticRuleFirst(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            if user.username == 'first':
+                return True
+            return False
+
+    class TestStaticRuleContainsS(AbstractBaseRule):
+        static = True
+
+        class Meta:
+            app_label = 'wagtail_personalisation'
+
+        def test_user(self, request, user):
+            if 's' in user.username:
+                return True
+            return False
+
+    django_user_model.objects.create(username='first')
+    django_user_model.objects.create(username='second')
+
+    segment = SegmentFactory.build(type=Segment.TYPE_STATIC)
+    first_rule = TestStaticRuleFirst()
+    s_rule = TestStaticRuleContainsS()
+    form = form_with_data(segment, first_rule, s_rule)
+
+    assert form.count_matching_users([first_rule, s_rule], False) is 1

tests/unit/test_utils.py

@@ -1,4 +1,5 @@
-from wagtail_personalisation.utils import impersonate_other_page
+from wagtail_personalisation.utils import (
+    exclude_variants, impersonate_other_page)
 
 
 class Page(object):
@@ -19,3 +20,40 @@ def test_impersonate_other_page():
     impersonate_other_page(page, other_page)
 
     assert page == other_page
+
+
+class Metadata(object):
+    def __init__(self, is_canonical=True):
+        self.is_canonical = is_canonical
+
+
+class PersonalisationMetadataPage(object):
+    def __init__(self):
+        self.personalisation_metadata = Metadata()
+
+
+def test_exclude_variants_includes_pages_with_no_metadata_property():
+    page = PersonalisationMetadataPage()
+    del page.personalisation_metadata
+    result = exclude_variants([page])
+    assert result == [page]
+
+
+def test_exclude_variants_includes_pages_with_metadata_none():
+    page = PersonalisationMetadataPage()
+    page.personalisation_metadata = None
+    result = exclude_variants([page])
+    assert result == [page]
+
+
+def test_exclude_variants_includes_pages_with_metadata_canonical():
+    page = PersonalisationMetadataPage()
+    result = exclude_variants([page])
+    assert result == [page]
+
+
+def test_exclude_variants_excludes_pages_with_metadata_not_canonical():
+    page = PersonalisationMetadataPage()
+    page.personalisation_metadata.is_canonical = False
+    result = exclude_variants([page])
+    assert result == []

tox.ini

@@ -20,7 +20,7 @@ commands =
 
 [testenv:lint]
 basepython = python2.7
-deps = flake8
+deps = flake8==3.5.0
 commands =
     flake8 src tests setup.py
     isort -q --recursive --diff src/ tests/