Update Elasticsearch

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7
+FROM python:3.9.22-bookworm
 
 ENV PYTHONUNBUFFERED 1
 
@@ -10,15 +10,21 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 
 # Copy, then install requirements before copying rest for a requirements cache layer.
-COPY requirements.txt /tmp/
+COPY requirements.txt patch_packages.py /tmp/
 RUN cd /tmp \
-    && pip install -r requirements.txt
+    && pip install --upgrade pip "setuptools<71" wheel \
+    && pip install -r requirements.txt \
+    && python /tmp/patch_packages.py
 
 COPY . /app
 
 WORKDIR /app
 
-RUN python manage.py compilemessages
+RUN DATABASE_URL=sqlite:///tmp/dummy.db \
+    ELASTICSEARCH_URL=http://localhost:9200 \
+    ELASTICSEARCH_INDEX=dummy \
+    DJANGO_SECRET_KEY=dummy-build-key \
+    python manage.py compilemessages
 
 RUN addgroup --system django \
     && adduser --system --ingroup django django \

bin/wait-for-deps.sh

@@ -6,4 +6,6 @@ set -o nounset
 source $(dirname $0)/wait-for-postgres.sh
 source $(dirname $0)/wait-for-elasticsearch.sh
 
+python /app/patch_packages.py
+
 exec "$@"

contactos/forms.py

@@ -1,7 +1,7 @@
 from django.forms import ModelForm
 from django.forms.models import ModelChoiceField
 from contactos.models import Contact
-from mailit import MailChannel
+from mailit.channel import MailChannel
 
 
 class ContactUpdateForm(ModelForm):

contactos/migrations/0001_initial.py

@@ -41,25 +41,25 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='contact',
             name='contact_type',
-            field=models.ForeignKey(to='contactos.ContactType'),
+            field=models.ForeignKey(to='contactos.ContactType', on_delete=models.CASCADE),
             preserve_default=True,
         ),
         migrations.AddField(
             model_name='contact',
             name='owner',
-            field=models.ForeignKey(related_name='contacts', to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='contacts', to=settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL),
             preserve_default=True,
         ),
         migrations.AddField(
             model_name='contact',
             name='person',
-            field=models.ForeignKey(to='popit.Person'),
+            field=models.ForeignKey(to='popit.Person', on_delete=models.CASCADE),
             preserve_default=True,
         ),
         migrations.AddField(
             model_name='contact',
             name='writeitinstance',
-            field=models.ForeignKey(related_name='contacts', to='instance.WriteItInstance', null=True),
+            field=models.ForeignKey(related_name='contacts', to='instance.WriteItInstance', null=True, on_delete=models.SET_NULL),
             preserve_default=True,
         ),
     ]

contactos/migrations/0002_contact_popolo_person.py

@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='contact',
             name='popolo_person',
-            field=models.ForeignKey(blank=True, to='popolo.Person', null=True),
+            field=models.ForeignKey(blank=True, to='popolo.Person', null=True, on_delete=models.SET_NULL),
             preserve_default=True,
         ),
     ]

contactos/migrations/0003_denull_popolo_people.py

@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='contact',
             name='popolo_person',
-            field=models.ForeignKey(to='popolo.Person'),
+            field=models.ForeignKey(to='popolo.Person', on_delete=models.CASCADE),
             preserve_default=True,
         ),
     ]

contactos/models.py

@@ -20,12 +20,12 @@ def __unicode__(self):
 
 class Contact(models.Model):
     """docstring for Contact"""
-    contact_type = models.ForeignKey('ContactType')
-    person = models.ForeignKey(Person)
+    contact_type = models.ForeignKey('ContactType', on_delete=models.CASCADE)
+    person = models.ForeignKey(Person, on_delete=models.CASCADE)
     value = models.CharField(max_length=512)
     is_bounced = models.BooleanField(default=False)
-    owner = models.ForeignKey(User, related_name="contacts", null=True)
-    writeitinstance = models.ForeignKey('instance.WriteItInstance', related_name="contacts", null=True)
+    owner = models.ForeignKey(User, related_name="contacts", null=True, on_delete=models.SET_NULL)
+    writeitinstance = models.ForeignKey('instance.WriteItInstance', related_name="contacts", null=True, on_delete=models.SET_NULL)
     enabled = models.BooleanField(default=True)
 
     def __unicode__(self):

contactos/urls.py

@@ -1,11 +1,11 @@
-from django.conf.urls import patterns, url
+from django.conf.urls import url
 from contactos.views import ContactoUpdateView, ContactCreateView
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'^contacto/update/(?P<pk>[-\d]+)/?$',
         ContactoUpdateView.as_view(),
         name='contact_value_update'),
     url(r'^(?P<pk>[-\d]+)/(?P<person_pk>[-\d]+)/contacto/create/?$',
         ContactCreateView.as_view(),
         name='create-new-contact'),
-)
+]

docker-compose.yml

@@ -74,11 +74,12 @@ services:
       - db-data:/var/lib/postgresql/data
 
   elasticsearch:
-    image: elasticsearch:1
+    image: elasticsearch:7.17.27
     volumes:
       - elasticsearch-data:/usr/share/elasticsearch/data
     environment:
-      - ES_MAX_MEM=1g
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms512m -Xmx1g
     ports:
       - 9200
 

global_test_case.py

@@ -72,7 +72,7 @@ def tearDown(self, *args, **kwargs):
         super(UsingDbMixin, self).tearDown(*args, **kwargs)
 
 
-from urlparse import urlparse
+from urllib.parse import urlparse
 
 
 def get_path_and_subdomain(path, **extra):
@@ -156,11 +156,10 @@ def setUp(self):
         super(SearchIndexTestCase, self).setUp()
         call_command('rebuild_index', verbosity=0, interactive=False)
 
-from djcelery.contrib.test_runner import CeleryTestSuiteRunner
 from django_nose import NoseTestSuiteRunner
 
 
-class WriteItTestRunner(CeleryTestSuiteRunner, NoseTestSuiteRunner):
+class WriteItTestRunner(NoseTestSuiteRunner):
     def run_tests(self, test_labels, extra_tests=None, **kwargs):
 
         # don't show logging messages while testing

instance/management/commands/instance_remove_extraneous_popolo_uri.py

@@ -19,7 +19,7 @@ def handle(self, *args, **options):
             if original_count == 0:
                 msg = "There were no popolo_uri Identifier objects for person " \
                       "{person} with ID {person_id}"
-                print msg.format(person=person, person_id=person.id)
+                print(msg.format(person=person, person_id=person.id))
                 errors_found = True
                 continue
             if original_count == 1:
@@ -28,10 +28,10 @@ def handle(self, *args, **options):
                     msg = "The only remaining popolo_uri Identifier for " \
                           "person {person} with ID {person_id} was a " \
                           "malformed legacy identifier: {bad_identifier}"
-                    print msg.format(
+                    print(msg.format(
                         person=person,
                         person_id=person.id,
-                        bad_identifier=sole_identifier.identifier)
+                        bad_identifier=sole_identifier.identifier))
                     errors_found = True
                     continue
             # Otherwise we have more than one identifier. Delete any
@@ -43,9 +43,9 @@ def handle(self, *args, **options):
             if len(unique_remaining) > 1:
                 msg = "There were multiple conflicting IDs for person " \
                       "{person} with ID {person_id}"
-                print msg.format(person=person, person_id=person.id)
+                print(msg.format(person=person, person_id=person.id))
                 for non_unique_id in sorted(unique_remaining):
-                    print " ", non_unique_id
+                    print(" ", non_unique_id)
                 errors_found = True
                 continue
             # Now remove all but one of the identifiers:

instance/migrations/0001_initial.py

@@ -19,7 +19,7 @@ class Migration(migrations.Migration):
             name='Membership',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('person', models.ForeignKey(to='popit.Person')),
+                ('person', models.ForeignKey(to='popit.Person', on_delete=models.CASCADE)),
             ],
             options={
             },
@@ -31,8 +31,8 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('name', models.CharField(max_length=255)),
                 ('description', models.CharField(max_length=512, blank=True)),
-                ('slug', autoslug.fields.AutoSlugField(populate_from=b'name', unique=True, editable=False)),
-                ('owner', models.ForeignKey(related_name='writeitinstances', to=settings.AUTH_USER_MODEL)),
+                ('slug', autoslug.fields.AutoSlugField(populate_from='name', unique=True, editable=False)),
+                ('owner', models.ForeignKey(related_name='writeitinstances', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
                 ('persons', models.ManyToManyField(related_name='writeit_instances', through='instance.Membership', to='popit.Person')),
             ],
             options={
@@ -56,10 +56,10 @@ class Migration(migrations.Migration):
                 ('email_port', models.IntegerField(null=True, blank=True)),
                 ('email_use_tls', models.NullBooleanField()),
                 ('email_use_ssl', models.NullBooleanField()),
-                ('can_create_answer', models.BooleanField(default=False, help_text=b'Can create an answer using the WebUI')),
+                ('can_create_answer', models.BooleanField(default=False, help_text='Can create an answer using the WebUI')),
                 ('maximum_recipients', models.PositiveIntegerField(default=5)),
-                ('default_language', models.CharField(max_length=10, choices=[(b'ar', b'Arabic'), (b'cs', b'Czech'), (b'en', b'English'), (b'es', b'Spanish'), (b'fr', b'French'), (b'hu', b'Hungarian')])),
-                ('writeitinstance', annoying.fields.AutoOneToOneField(related_name='config', to='instance.WriteItInstance')),
+                ('default_language', models.CharField(max_length=10, choices=[('ar', 'Arabic'), ('cs', 'Czech'), ('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('hu', 'Hungarian')])),
+                ('writeitinstance', annoying.fields.AutoOneToOneField(related_name='config', to='instance.WriteItInstance', on_delete=models.CASCADE)),
             ],
             options={
             },
@@ -69,13 +69,13 @@ class Migration(migrations.Migration):
             name='WriteitInstancePopitInstanceRecord',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('periodicity', models.CharField(default=b'1W', max_length=b'2', choices=[(b'--', b'Never'), (b'2D', b'Twice every Day'), (b'1D', b'Daily'), (b'1W', b'Weekly')])),
-                ('status', models.CharField(default=b'nothing', max_length=b'20', choices=[(b'nothing', 'Not doing anything now'), (b'error', 'Error'), (b'success', 'Success'), (b'waiting', 'Waiting'), (b'inprogress', 'In Progress')])),
-                ('status_explanation', models.TextField(default=b'')),
+                ('periodicity', models.CharField(default='1W', max_length=2, choices=[('--', 'Never'), ('2D', 'Twice every Day'), ('1D', 'Daily'), ('1W', 'Weekly')])),
+                ('status', models.CharField(default='nothing', max_length=20, choices=[('nothing', 'Not doing anything now'), ('error', 'Error'), ('success', 'Success'), ('waiting', 'Waiting'), ('inprogress', 'In Progress')])),
+                ('status_explanation', models.TextField(default='')),
                 ('updated', models.DateTimeField(auto_now_add=True)),
                 ('created', models.DateTimeField(auto_now_add=True)),
-                ('popitapiinstance', models.ForeignKey(to='popit.ApiInstance')),
-                ('writeitinstance', models.ForeignKey(to='instance.WriteItInstance')),
+                ('popitapiinstance', models.ForeignKey(to='popit.ApiInstance', on_delete=models.CASCADE)),
+                ('writeitinstance', models.ForeignKey(to='instance.WriteItInstance', on_delete=models.CASCADE)),
             ],
             options={
             },
@@ -84,7 +84,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='membership',
             name='writeitinstance',
-            field=models.ForeignKey(to='instance.WriteItInstance'),
+            field=models.ForeignKey(to='instance.WriteItInstance', on_delete=models.CASCADE),
             preserve_default=True,
         ),
     ]

instance/migrations/0003_add_parallel_popolo_data.py

@@ -17,8 +17,8 @@ class Migration(migrations.Migration):
             name='InstanceMembership',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('person', models.ForeignKey(to='popolo.Person')),
-                ('writeitinstance', models.ForeignKey(to='instance.WriteItInstance')),
+                ('person', models.ForeignKey(to='popolo.Person', on_delete=models.CASCADE)),
+                ('writeitinstance', models.ForeignKey(to='instance.WriteItInstance', on_delete=models.CASCADE)),
             ],
             options={
             },
@@ -33,7 +33,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='writeitinstancepopitinstancerecord',
             name='popolo_source',
-            field=models.ForeignKey(blank=True, to='popolo_sources.PopoloSource', null=True),
+            field=models.ForeignKey(blank=True, to='popolo_sources.PopoloSource', null=True, on_delete=models.SET_NULL),
             preserve_default=True,
         ),
     ]

instance/migrations/0004_add_django_popolo_people.py

@@ -2,10 +2,9 @@
 from __future__ import unicode_literals
 
 import re
-from urlparse import urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
 
 from django.db import migrations
-from django.contrib.contenttypes.management import update_contenttypes
 
 
 def is_proxy_url(url):
@@ -38,14 +37,11 @@ def update_source_url(original_url):
 
 
 def forwards(apps, schema_editor):
-    # Make sure the content types for django-popolo exist, with a
-    # hacky workaround from: http://stackoverflow.com/a/35353170/223092
-    popolo_app = apps.app_configs['popolo']
-    popolo_app.models_module = popolo_app.models_module or True
-    update_contenttypes(popolo_app, verbosity=1, interactive=False)
+    # Make sure the content type for popolo Person exists
     ContentType = apps.get_model('contenttypes', 'ContentType')
-    person_content_type = ContentType.objects.get(
-        app_label='popolo', model='person')
+    person_content_type, _ = ContentType.objects.get_or_create(
+        app_label='popolo', model='person',
+        defaults={'app_label': 'popolo', 'model': 'person'})
     # Create a PopoloSource for each old APIInstance
     ApiInstance = apps.get_model('popit', 'ApiInstance')
     PopoloSource = apps.get_model('popolo_sources', 'PopoloSource')

instance/migrations/0004_add_email_real_name_to_config.py

@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='writeitinstanceconfig',
             name='real_name_for_site_emails',
-            field=models.TextField(default=b'', help_text='The name that should appear in the From: line of emails sent from this site', blank=True),
+            field=models.TextField(default='', help_text='The name that should appear in the From: line of emails sent from this site', blank=True),
             preserve_default=True,
         ),
     ]

instance/migrations/0005_denull_popolo_source.py

@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='writeitinstancepopitinstancerecord',
             name='popolo_source',
-            field=models.ForeignKey(to='popolo_sources.PopoloSource'),
+            field=models.ForeignKey(to='popolo_sources.PopoloSource', on_delete=models.CASCADE),
             preserve_default=True,
         ),
     ]

instance/migrations/0010_add_popoloperson_proxy_and_change_related.py

@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='instancemembership',
             name='person',
-            field=models.ForeignKey(to='instance.PopoloPerson'),
+            field=models.ForeignKey(to='instance.PopoloPerson', on_delete=models.CASCADE),
             preserve_default=True,
         ),
         migrations.AlterField(

instance/migrations/0011_update_default_language_choice.py

@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='writeitinstanceconfig',
             name='default_language',
-            field=models.CharField(max_length=10, choices=[(b'ar', b'Arabic'), (b'cs', b'Czech'), (b'en', b'English'), (b'es', b'Spanish'), (b'fa', b'Persian'), (b'fr', b'French'), (b'hu', b'Hungarian')]),
+            field=models.CharField(max_length=10, choices=[('ar', 'Arabic'), ('cs', 'Czech'), ('en', 'English'), ('es', 'Spanish'), ('fa', 'Persian'), ('fr', 'French'), ('hu', 'Hungarian')]),
             preserve_default=True,
         ),
     ]