Merge pull request #124 from cfpb/cache-total-accts

Save total accounts per event, provide to API

Author: schbetsy

django/evaluate_m2/tests/test_views.py

@@ -116,7 +116,8 @@ def create_activity_data(self, create_zero_hit:bool=False):
             id=1, name='test_exam', portfolio='credit cards',
             directory='Enforcement/Huyndai2025', eid_or_matter_num='123-456789',
             other_descriptor='',date_range_start='2023-11-30',
-            date_range_end='2023-12-31', total_tradelines=1234)
+            date_range_end='2023-12-31', total_tradelines=1234,
+            total_accounts=321)
         self.data_file = M2DataFile.objects.create(
             event=self.event, file_name='file.txt'
         )
@@ -437,6 +438,7 @@ def test_events_view(self):
             'date_range_start': '2023-11-30',
             'date_range_end': '2023-12-31',
             'total_tradelines': 1234,
+            'total_accounts': 321,
             'evaluators': [{
                 'hits': 2,
                 'accounts_affected': 1,

django/evaluate_m2/views.py

@@ -198,6 +198,7 @@ def events_view(request, event_id):
             "date_range_start": event.date_range_start,
             "date_range_end": event.date_range_end,
             "total_tradelines": event.total_tradelines,
+            "total_accounts": event.total_accounts,
             "evaluators": evaluator_metadata_serializer.data,
         }
         return Response(result)

django/parse_m2/admin.py

@@ -52,6 +52,7 @@ class Metro2EventAdmin(Metro2AdminPermissions):
         'date_range_start',
         'date_range_end',
         'total_tradelines',
+        'total_accounts',
         'prior_records_associated',
     ]
     fields = [
@@ -64,6 +65,7 @@ class Metro2EventAdmin(Metro2AdminPermissions):
         'date_range_start',
         'date_range_end',
         'total_tradelines',
+        'total_accounts',
         'prior_records_associated',
     ]
     list_display = [
@@ -74,7 +76,6 @@ class Metro2EventAdmin(Metro2AdminPermissions):
         'date_range_start',
         'date_range_end',
         'total_tradelines',
-        'prior_records_associated',
     ]
     filter_horizontal = ['members']
 

django/parse_m2/initiate_post_parsing.py

@@ -12,6 +12,8 @@ def post_parse(event: Metro2Event, strategy:str = 'chunk') -> None:
     logger.info("Calculating total records.")
     save_total_records(event)
     if event.total_tradelines > 0:
+        logger.info("Calculating total accounts.")
+        save_total_accts(event)
         logger.info("Calculating event date range.")
         save_date_range(event)
         logger.info("Beginning progressive evaluator query.")
@@ -27,6 +29,11 @@ def save_date_range(event: Metro2Event):
     event.date_range_end = date_range['latest']
     event.save()
 
+def save_total_accts(event: Metro2Event):
+    event.total_accounts = event.accountactivity_set.values(
+        'cons_acct_num').distinct().count()
+    event.save()
+
 def save_total_records(event: Metro2Event):
     event.total_tradelines = event.calculate_total_tradelines()
     event.save()
@@ -54,7 +61,7 @@ def associate_previous_records(event: Metro2Event, strategy: str = 'chunk'):
         except ValueError:
             chunk_method_associate_prior_records(event)
     elif strategy == 'skip':
-        logger.info("'Skip' option, was used; skipping prior records step.")
+        logger.info("'Skip' option selected; skipping prior records step.")
     else:
         logger.info(
             f"Invalid strategy input: {strategy}. "
@@ -66,12 +73,12 @@ def report_on_prior_record_outcome(event: Metro2Event):
     record_set = event.get_all_account_activity()
     total_updated = record_set.filter(previous_values_id__isnull=False).count()
     logger.info(f"Records with a previous record associated: {total_updated:,}")
-    total_not_updated = record_set.filter(previous_values_id__isnull=True).count()
-    logger.info(f"Records with NO previous record associated: {total_not_updated:,}")
-
     event.prior_records_associated = total_updated
     event.save()
 
+    total_not_updated = record_set.filter(previous_values_id__isnull=True).count()
+    logger.info(f"Records with NO previous record associated: {total_not_updated:,}")
+
 
 # Chunk strategy for associating prior records
 ##########################################################################
@@ -84,7 +91,11 @@ def report_on_prior_record_outcome(event: Metro2Event):
 def chunk_method_associate_prior_records(event: Metro2Event, chunk_size = 100_000):
     logger = logging.getLogger('parse_m2.chunk_method_associate_prior_records')
     # Count the number of distinct account numbers
-    total_accts = event.accountactivity_set.values('cons_acct_num').distinct().count()
+    if event.total_accounts:
+        total_accts = event.total_accounts
+    else:
+        total_accts = event.accountactivity_set.values(
+            'cons_acct_num').distinct().count()
 
     # Use that to decide number of chunks
     q,r = divmod(total_accts, chunk_size)

django/parse_m2/management/commands/delete_event_data.py

@@ -63,6 +63,14 @@ def handle(self, *args, **options):
             logger.info(f"Deleting {parsed_files.count()} existing files.")
             parsed_files.delete()
 
+        logger.info("Deleting any cached summary data for event")
+        event.date_range_start = None
+        event.date_range_end = None
+        event.total_tradelines = 0
+        event.total_accounts = 0
+        event.prior_records_associated = 0
+        event.save()
+
         logger.info(
             self.style.SUCCESS(f"Finished deleting data for event: {event_id}.")
         )

django/parse_m2/migrations/0007_metro2event_total_accounts.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.17 on 2026-10-13 21:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parse_m2', '0006_metro2event_prior_records_associated'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='metro2event',
+            name='total_accounts',
+            field=models.IntegerField(default=0),
+        ),
+    ]

django/parse_m2/models.py

@@ -24,6 +24,7 @@ class Meta:
     date_range_start = models.DateField(null=True)
     date_range_end = models.DateField(null=True)
     total_tradelines = models.IntegerField(default=0)
+    total_accounts = models.IntegerField(default=0)
     prior_records_associated = models.IntegerField(default=0)
 
     def __str__(self) -> str:

django/parse_m2/tests/test_post_parsing.py

@@ -41,6 +41,11 @@ def test_post_parse_total_tradelines(self):
         # post_parse updates total_tradelines value
         self.assertEqual(self.event.total_tradelines, 6)
 
+    def test_post_parse_total_accounts(self):
+        self.assertEqual(self.event.total_accounts, 0)
+        post_parse(self.event)
+        self.assertEqual(self.event.total_accounts, 2)
+
     def test_associate_previous_records_no_previous_records(self):
         associate_previous_records(self.event)
         # Retrieve any record with activity_date 2018-01-31