diff --git a/common/cloudflare.py b/common/cloudflare.py index 33c2a7f9..cf68d106 100644 --- a/common/cloudflare.py +++ b/common/cloudflare.py @@ -5,7 +5,6 @@ import requests from django.conf import settings -from django.core.cache import cache from django.contrib.sites.models import Site from django.urls import reverse @@ -53,12 +52,7 @@ def purge_vis_cache(cls, slug): @classmethod def purge_paths_cache(cls, paths): - """ Purges the URLs (paths, not URLs) """ - # We also want to purge the file-based cache, but unfortunately - # we don't have a way of doing this per-URL. - # It's overkill, but here we purge everything. - cache.clear() - + """ Purges the Cloudflare CDN cache for the given paths. """ # If we're on local/dev/staging/etc, we're done. if not cls._is_api_enabled(): return diff --git a/movie/models.py b/movie/models.py index c5f2e95e..028c45b7 100644 --- a/movie/models.py +++ b/movie/models.py @@ -1,6 +1,5 @@ """ Models for storing data about a movie """ from django.conf import settings -from django.core.cache import cache from django.core.files.storage import DefaultStorage from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models @@ -28,10 +27,6 @@ class Movie(models.Model): def save(self, *args, **kwargs): super().save(*args, **kwargs) - # Clear the cache. Otherwise, you'll continue to get the cached result - # of the old model. - cache.clear() - class TextToSpeechCachedFile(models.Model): """ A mapping from a text to an audio file of the text-to-speech mp3 """ diff --git a/rcvis/settings.py b/rcvis/settings.py index b75da902..d480e607 100644 --- a/rcvis/settings.py +++ b/rcvis/settings.py @@ -81,10 +81,8 @@ 'django.contrib.sessions.middleware.SessionMiddleware', - # Order of the next 3 is important - 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.common.CommonMiddleware', - 'django.middleware.cache.FetchFromCacheMiddleware', + 'django.middleware.http.ConditionalGetMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -281,20 +279,14 @@ AWS_DEFAULT_ACL = None -if os.environ.get('DISABLE_CACHE') != 'True': - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/tmp/django_rcvis_cache/', - } - } -else: - assert DEBUG - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - } +# In-memory cache for lightweight uses (e.g. upload rate limiting). +# Full HTTP response caching is handled by Cloudflare at the edge; +# Django's cache middleware has been removed. +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', } +} REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, diff --git a/scripts/reset-db.sh b/scripts/reset-db.sh new file mode 100755 index 00000000..24f378fa --- /dev/null +++ b/scripts/reset-db.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Reset the SQLite database and re-run all migrations. +# Use after switching branches with incompatible migration history. +set -e + +source venv/bin/activate +source .env + +rm -f db.sqlite3 +python manage.py migrate + +# Create API-enabled admin user (matches docker-entrypoint.sh) +python manage.py shell -c " +from django.contrib.auth import get_user_model +User = get_user_model() +if not User.objects.filter(username='skaphan').exists(): + user = User.objects.create_superuser('skaphan', 'sjk@kaphan.org', 'rcvisacc0unt') + user.userprofile.canUseApi = True + user.userprofile.save() + print('Created API user skaphan with API access') +else: + print('API user skaphan already exists') +" + +echo "Database reset complete." diff --git a/visualizer/migrations/0033_add_updated_at.py b/visualizer/migrations/0033_add_updated_at.py new file mode 100644 index 00000000..4b8255fa --- /dev/null +++ b/visualizer/migrations/0033_add_updated_at.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-02-22 12:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('visualizer', '0032_jsonconfig_forcefirstrounddeterminespercentages'), + ] + + operations = [ + migrations.AddField( + model_name='jsonconfig', + name='updatedAt', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/visualizer/models.py b/visualizer/models.py index 789dba60..65a6a4e1 100644 --- a/visualizer/models.py +++ b/visualizer/models.py @@ -50,6 +50,7 @@ class JsonConfig(models.Model): candidateSidecarFile = models.FileField(null=True, blank=True) slug = models.SlugField(unique=True, max_length=255) uploadedAt = models.DateTimeField(auto_now_add=True) + updatedAt = models.DateTimeField(auto_now=True) owner = models.ForeignKey( settings.AUTH_USER_MODEL, related_name='this_users_jsons', diff --git a/visualizer/tests/testLiveBrowserHeadless.py b/visualizer/tests/testLiveBrowserHeadless.py index e4d290ec..a57b7e76 100644 --- a/visualizer/tests/testLiveBrowserHeadless.py +++ b/visualizer/tests/testLiveBrowserHeadless.py @@ -12,10 +12,10 @@ from django.conf import settings from django.contrib.auth import get_user_model -from django.core.cache import cache + from django.core import mail as test_mailbox from django.urls import reverse -from mock import patch + from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -23,7 +23,7 @@ from selenium.webdriver.support.ui import WebDriverWait from common.testUtils import TestHelpers -from common.viewUtils import get_data_for_view + from visualizer.models import TextForWinner from visualizer.tests import filenames from visualizer.tests import liveServerTestBaseClass @@ -57,16 +57,6 @@ def _go_to_without_cache(self, url): return localBrowser - def _assert_cache_is_running(self): - """ - Ensure bmemcached is running - always working with the current infra (filestyem-backed) - but other types (memcached) need to be started before tests run - """ - cache.set('key', 'value') - if not cache.get('key') == 'value': - print("You must start bmemcached before running these tests") - self.assertEqual(cache.get('key'), 'value') - def test_upload(self): """ Tests the upload page """ self.open('/upload.html') @@ -211,88 +201,6 @@ def test_legacy_embedded_urls(self): except NoSuchElementException: pass - def test_cache_speed(self): - """ Tests that caching works and that second loads are faster, - even without browser cache """ - - # Verify that the django.core.cache middleware works as expected - def measure_load_time(localBrowser, url): - localBrowser.get(self._make_url(url)) - - WebDriverWait(localBrowser, timeout=5, poll_frequency=0.05).until( - lambda d: d.find_element(By.ID, "page-top")) - - tic = localBrowser.execute_script('return performance.timing.fetchStart') - toc = localBrowser.execute_script('return performance.timing.domLoading') - return toc - tic - - def is_cache_much_faster(baseUrl, shouldItBe): - urlsToLoad = [f"{baseUrl}?a={num}" for num in range(3)] - urlToInitializeBrowserCache = baseUrl + "?just_to_init_browser_cache=1" - - # First, go to something that sets up the browser cache with static files - localBrowser = self._go_to_without_cache(urlToInitializeBrowserCache) - - loadTimesWithoutCache = [measure_load_time(localBrowser, f) for f in urlsToLoad] - loadTimesWithCache = [measure_load_time(localBrowser, f) for f in urlsToLoad] - - avgLoadTimeWithoutCache = sum(loadTimesWithoutCache) / len(loadTimesWithoutCache) - avgLoadTimeWithCache = sum(loadTimesWithCache) / len(loadTimesWithCache) - - # Verify that it's at least 2x faster with cache (closer to 5x on - # selenium, 200x in real life) - print("For debugging this flaky test: Without cache", loadTimesWithoutCache) - print("For debugging this flaky test: With cache", loadTimesWithCache) - if shouldItBe: - self.assertGreater(avgLoadTimeWithoutCache, avgLoadTimeWithCache * 2) - else: - self.assertLess(avgLoadTimeWithoutCache, avgLoadTimeWithCache * 2) - - self._assert_cache_is_running() - - # Upload a file - self._upload_something_if_needed() - baseUrl = reverse('visualize', args=(TestHelpers.get_latest_upload().slug,)) - - # Force cache clearing - TestHelpers.get_latest_upload().save() - - # Initial load should not be cached - is_cache_much_faster(baseUrl, True) - - # Uploading should clear all cache - self._upload(filenames.ONE_ROUND) - is_cache_much_faster(baseUrl, True) - - # But just visiting the upload page and returning should not clear cache - self.open("/upload.html") - is_cache_much_faster(baseUrl, False) - - @patch('common.viewUtils.get_data_for_view', side_effect=get_data_for_view) - def test_cache_works(self, dataForViewPatch): - """ Tests that caching doesn't call the heavy graph-generation function more than once """ - def count_cache_misses_mocked(url): - dataForViewPatch.reset_mock() - self._go_to_without_cache(url) - return dataForViewPatch.call_count - - self._assert_cache_is_running() - - self._upload_something_if_needed() - url = reverse('visualize', args=(TestHelpers.get_latest_upload().slug,)) - - # Updating should clear all cache - TestHelpers.get_latest_upload().save() - self.assertEqual(count_cache_misses_mocked(url), 1) - - # Going to the same URL should now be cached - self.assertEqual(count_cache_misses_mocked(url), 0) - - # Uploading also clears cache, but this is accidental - - # it just happens to save() multiple times - self._upload(filenames.ONE_ROUND) - self.assertEqual(count_cache_misses_mocked(url), 1) - def test_sharetab_sane_links(self): """ Check that the share tab has sane links for all buttons """ self._upload_something_if_needed() diff --git a/visualizer/views.py b/visualizer/views.py index d1b51011..ca430d4f 100644 --- a/visualizer/views.py +++ b/visualizer/views.py @@ -12,7 +12,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.mixins import LoginRequiredMixin from django.core.cache import cache -from django.http import JsonResponse, HttpResponse +from django.http import JsonResponse, HttpResponse, HttpResponseNotModified from django.shortcuts import render from django.templatetags.static import static from django.urls import Resolver404 @@ -21,7 +21,8 @@ from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.clickjacking import xframe_options_exempt -from django.views.decorators.vary import vary_on_headers +from django.utils.cache import patch_cache_control +from django.utils.http import http_date, parse_http_date_safe from django.views.generic.base import TemplateView, RedirectView from django.views.generic.detail import DetailView from django.views.generic.edit import CreateView @@ -160,8 +161,42 @@ def _actions_before_save(self, form): self.model.jsonFile.save('datatablesfile.json', form.cleaned_data['jsonFile']) -@method_decorator(vary_on_headers('increment',), name='get') -class Visualize(DetailView): +class ConditionalGetMixin: # pylint: disable=too-few-public-methods + """ + Mixin for DetailView subclasses that serve JsonConfig visualizations. + Short-circuits with 304 Not Modified when the client's If-Modified-Since + matches the object's updatedAt, skipping expensive graph computation + and template rendering. Also sets Last-Modified and Cache-Control on + all responses. + """ + + def get(self, request, *args, **kwargs): + """Return 304 if the client's copy is fresh, otherwise render normally.""" + # Fetch object once — setting self.object avoids a second DB query + # when super().get() calls get_object() internally. + self.object = self.get_object() + + # Short-circuit: if the client has a fresh copy, return 304 without + # doing any of the expensive graph computation or template rendering. + if self.object.updatedAt: + lastModified = self.object.updatedAt.timestamp() + ifModifiedSince = request.META.get('HTTP_IF_MODIFIED_SINCE') + if ifModifiedSince: + ifModifiedSince = parse_http_date_safe(ifModifiedSince) + if ifModifiedSince is not None and lastModified <= ifModifiedSince: + response = HttpResponseNotModified() + response['Last-Modified'] = http_date(lastModified) + patch_cache_control(response, no_cache=True, max_age=0) + return response + + response = super().get(request, *args, **kwargs) + if self.object.updatedAt: + response['Last-Modified'] = http_date(self.object.updatedAt.timestamp()) + patch_cache_control(response, no_cache=True, max_age=0) + return response + + +class Visualize(ConditionalGetMixin, DetailView): """ Visualizing a single JsonConfig """ model = JsonConfig template_name = 'visualizer/visualize.html' @@ -198,9 +233,8 @@ def get_context_data(self, **kwargs): return data -@method_decorator(vary_on_headers('increment',), name='get') @method_decorator(xframe_options_exempt, name='dispatch') -class VisualizeEmbedded(DetailView): +class VisualizeEmbedded(ConditionalGetMixin, DetailView): """ The embedded visualization, to be used in an iframe. """ @@ -258,9 +292,8 @@ def get_redirect_url(self, *args, **kwargs): return super().get_redirect_url(slug) + "?vistype=" + vistype -@method_decorator(vary_on_headers('increment',), name='get') @method_decorator(xframe_options_exempt, name='dispatch') -class VisualizeBallotpedia(DetailView): +class VisualizeBallotpedia(ConditionalGetMixin, DetailView): """ The embedded ballotpedia visualization """ model = JsonConfig template_name = 'visualizer/visualize-ballotpedia.html'