From a8a068e8c831d05029c58c908679235d5a7ccd3d Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Tue, 13 Feb 2024 14:13:11 -0500 Subject: [PATCH 01/10] Update pages with multiple graphs to scroll to shared graph --- .../Charts/ChartSections/ShareList.js | 10 ++++--- .../Charts/Contraband/Contraband.js | 21 ++++++++++++--- .../Components/Charts/Searches/Searches.js | 21 ++++++++++++--- .../Charts/TrafficStops/TrafficStops.js | 26 ++++++++++++++++--- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/frontend/src/Components/Charts/ChartSections/ShareList.js b/frontend/src/Components/Charts/ChartSections/ShareList.js index f6a88b5d..6c74e6ed 100644 --- a/frontend/src/Components/Charts/ChartSections/ShareList.js +++ b/frontend/src/Components/Charts/ChartSections/ShareList.js @@ -5,11 +5,15 @@ import * as S from './ShareList.styled'; import TwitterLogo from '../../../img/x-logo-black.png'; import FacebookLogo from '../../../img/meta_logo_primary.svg'; -function ShareList({ shareUrl, twitterTitle, onPressHandler }) { +function ShareList({ shareUrl, twitterTitle, onPressHandler, graphAnchor = null }) { + let shareURL = shareUrl; + if (graphAnchor) { + shareURL = `${shareURL}#${graphAnchor}`; + } return ( { + if (window.location.hash) { + document.querySelector(`${window.location.hash}`).scrollIntoView(); + } + }, []); + const [year, setYear] = useState(YEARS_DEFAULT); const renderMetaTags = useMetaTags(); @@ -607,10 +613,13 @@ function Contraband(props) { - +

@@ -670,10 +679,13 @@ function Contraband(props) { - + setContrabandTypesData((state) => ({ ...state, isOpen: true }))} + shareProps={{ + graphAnchor: 'hit_rate_by_type', + }} />

Shows what percentage of searches discovered specific types of illegal items.

@@ -720,12 +732,15 @@ function Contraband(props) { - + setGroupedContrabandStopPurposeModalData((state) => ({ ...state, isOpen: true })) } + shareProps={{ + graphAnchor: 'hit_rate_by_type_and_stop_purpose', + }} />

diff --git a/frontend/src/Components/Charts/Searches/Searches.js b/frontend/src/Components/Charts/Searches/Searches.js index 4219629c..65fc5540 100644 --- a/frontend/src/Components/Charts/Searches/Searches.js +++ b/frontend/src/Components/Charts/Searches/Searches.js @@ -38,6 +38,12 @@ function Searches(props) { useDataset(agencyId, SEARCHES); const [chartState] = useDataset(agencyId, SEARCHES_BY_TYPE); + useEffect(() => { + if (window.location.hash) { + document.querySelector(`${window.location.hash}`).scrollIntoView(); + } + }, []); + const [searchType, setSearchType] = useState(SEARCH_TYPE_DEFAULT); const [searchPercentageData, setSearchPercentageData] = useState({ @@ -161,10 +167,13 @@ function Searches(props) { {/* Search Rate */} {renderMetaTags()} {renderTableModal()} - +

Shows the percent of stops that led to searches, broken down by race/ethnicity.

@@ -192,8 +201,14 @@ function Searches(props) { {/* Searches by Count */} - - + +

Shows the number of searches performed {subjectObserving()}, broken down by search type and race / ethnicity. diff --git a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js index 0380ea9b..95eec30b 100644 --- a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js +++ b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js @@ -60,6 +60,12 @@ function TrafficStops(props) { // TODO: Remove this when moving table modal data to new modal component useDataset(agencyId, STOPS_BY_REASON); + useEffect(() => { + if (window.location.hash) { + document.querySelector(`${window.location.hash}`).scrollIntoView(); + } + }, []); + const [pickerActive, setPickerActive] = useState(null); const [year, setYear] = useState(YEARS_DEFAULT); @@ -663,8 +669,14 @@ function TrafficStops(props) { {/* Traffic Stops by Count */} - - + +

Shows the number of traffics stops broken down by purpose and race / ethnicity.

@@ -706,10 +718,13 @@ function TrafficStops(props) { - +

Shows the number of traffics stops broken down by purpose and race / ethnicity.

 - +

Shows the number of traffics stops broken down by purpose and race / ethnicity.

From 7f9efaa3f82a49537146d0ddcfe406e22f1639de Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Wed, 14 Feb 2024 11:43:46 -0500 Subject: [PATCH 02/10] Update overview page to call year range from util func instead of context state --- frontend/src/Components/Charts/Overview/Overview.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/Components/Charts/Overview/Overview.js b/frontend/src/Components/Charts/Overview/Overview.js index 37209b6a..301d49d8 100644 --- a/frontend/src/Components/Charts/Overview/Overview.js +++ b/frontend/src/Components/Charts/Overview/Overview.js @@ -32,12 +32,14 @@ import useOfficerId from '../../../Hooks/useOfficerId'; import DataSubsetPicker from '../ChartSections/DataSubsetPicker/DataSubsetPicker'; import PieChart from '../../NewCharts/PieChart'; import { pieChartConfig, pieChartLabels } from '../../../util/setChartColors'; +import useYearSet from '../../../Hooks/useYearSet'; function Overview(props) { const { agencyId } = props; const history = useHistory(); const match = useRouteMatch(); const officerId = useOfficerId(); + const [yearRange] = useYearSet(); useDataset(agencyId, STOPS); useDataset(agencyId, SEARCHES); @@ -203,7 +205,7 @@ function Overview(props) { label="Year" value={year} onChange={handleYearSelect} - options={[YEARS_DEFAULT].concat(chartState.yearRange)} + options={[YEARS_DEFAULT].concat(yearRange)} dropDown /> From 44afb257beb8f21dc713478ba0c2e55496f84b25 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Wed, 14 Feb 2024 12:19:26 -0500 Subject: [PATCH 03/10] Consolidate Contraband year dropdowns (#269) * Consolidate contraband dropdowns to just one that updates all graphs --- .../Charts/Contraband/Contraband.js | 107 +++++------------- 1 file changed, 29 insertions(+), 78 deletions(-) diff --git a/frontend/src/Components/Charts/Contraband/Contraband.js b/frontend/src/Components/Charts/Contraband/Contraband.js index 624888c1..4637a4f0 100644 --- a/frontend/src/Components/Charts/Contraband/Contraband.js +++ b/frontend/src/Components/Charts/Contraband/Contraband.js @@ -55,7 +55,6 @@ function Contraband(props) { csvData: [], loading: true, }); - const [contrabandYear, setContrabandYear] = useState(YEARS_DEFAULT); const [contrabandTypesData, setContrabandTypesData] = useState({ labels: [], datasets: [], @@ -65,7 +64,6 @@ function Contraband(props) { loading: true, }); - const [contrabandTypesYear, setContrabandTypesYear] = useState(YEARS_DEFAULT); const [contrabandStopPurposeData, setContrabandStopPurposeData] = useState({ labels: [], datasets: [], @@ -89,8 +87,6 @@ function Contraband(props) { loading: true, }); - const [contrabandStopPurposeYear, setContrabandStopPurposeYear] = useState(YEARS_DEFAULT); - const initialContrabandGroupedData = [ { labels: [], @@ -153,25 +149,11 @@ function Contraband(props) { fetchHitRateByStopPurpose(y); }; - const handleContrabandYearSelect = (y) => { - if (y === contrabandYear) return; - setContrabandYear(y); - }; - const handleContrabandTypesYearSelect = (y) => { - if (y === contrabandTypesYear) return; - setContrabandTypesYear(y); - }; - - const handleGroupedContrabandYearSelect = (y) => { - if (y === contrabandStopPurposeYear) return; - setContrabandStopPurposeYear(y); - }; - // Build New Contraband Data useEffect(() => { const params = []; - if (contrabandYear && contrabandYear !== 'All') { - params.push({ param: 'year', val: contrabandYear }); + if (year && year !== 'All') { + params.push({ param: 'year', val: year }); } if (officerId) { params.push({ param: 'officer', val: officerId }); @@ -223,12 +205,12 @@ function Contraband(props) { setContrabandData(data); }) .catch((err) => console.log(err)); - }, [contrabandYear]); + }, [year]); useEffect(() => { const params = []; - if (contrabandTypesYear && contrabandTypesYear !== 'All') { - params.push({ param: 'year', val: contrabandTypesYear }); + if (year && year !== 'All') { + params.push({ param: 'year', val: year }); } if (officerId) { params.push({ param: 'officer', val: officerId }); @@ -279,12 +261,12 @@ function Contraband(props) { setContrabandTypesData(data); }) .catch((err) => console.log(err)); - }, [contrabandTypesYear]); + }, [year]); useEffect(() => { const params = []; - if (contrabandStopPurposeYear && contrabandStopPurposeYear !== 'All') { - params.push({ param: 'year', val: contrabandStopPurposeYear }); + if (year && year !== 'All') { + params.push({ param: 'year', val: year }); } if (officerId) { params.push({ param: 'officer', val: officerId }); @@ -320,7 +302,7 @@ function Contraband(props) { }); }) .catch((err) => console.log(err)); - }, [contrabandStopPurposeYear]); + }, [year]); useEffect(() => { fetchHitRateByStopPurpose('All'); @@ -517,14 +499,14 @@ function Contraband(props) { const getBarChartModalSubHeading = (title) => `${title} ${subjectObserving()}.`; - const getBarChartModalHeading = (title, yearSelected) => { + const getBarChartModalHeading = (title) => { let subject = chartState.data[AGENCY_DETAILS].name; if (officerId) { subject = `Officer ${officerId}`; } let fromYear = ` since ${chartState.yearRange[chartState.yearRange.length - 1]}`; - if (yearSelected && yearSelected !== 'All') { - fromYear = ` in ${yearSelected}`; + if (year && year !== 'All') { + fromYear = ` in ${year}`; } return `${title} by ${subject}${fromYear}`; }; @@ -563,6 +545,15 @@ function Contraband(props) { a tiny fraction of the illegal substance +
+ +
- - - @@ -662,21 +644,11 @@ function Contraband(props) { ), agencyName: chartState.data[AGENCY_DETAILS].name, chartTitle: getBarChartModalHeading( - 'Contraband "Hit Rate" Grouped By Stop Purpose', - contrabandStopPurposeYear + 'Contraband "Hit Rate" Grouped By Stop Purpose' ), }} /> - - - @@ -714,22 +686,11 @@ function Contraband(props) { 'Shows what number of searches discovered specific types of illegal items' ), agencyName: chartState.data[AGENCY_DETAILS].name, - chartTitle: getBarChartModalHeading( - 'Contraband "Hit Rate" by type', - contrabandTypesYear - ), + chartTitle: getBarChartModalHeading('Contraband "Hit Rate" by type'), }} /> - - - + @@ -792,13 +753,6 @@ function Contraband(props) { ))} - @@ -843,8 +796,7 @@ function Contraband(props) { ), agencyName: chartState.data[AGENCY_DETAILS].name, chartTitle: getBarChartModalHeading( - 'Contraband "Hit Rate" by Type grouped by Regulatory/Equipment', - year + 'Contraband "Hit Rate" by Type grouped by Regulatory/Equipment' ), }} /> @@ -870,8 +822,7 @@ function Contraband(props) { ), agencyName: chartState.data[AGENCY_DETAILS].name, chartTitle: getBarChartModalHeading( - 'Contraband "Hit Rate" by Type grouped by Other', - year + 'Contraband "Hit Rate" by Type grouped by Other' ), }} /> From 63960dfa4f8d9b5c01af79168688c712694794ea Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Wed, 21 Feb 2024 17:55:42 -0500 Subject: [PATCH 04/10] Add support for caching api/mat views. (#271) * update new api endpoints to cache response for a day and update mat views to use cache mixin --- nc/data/importer.py | 4 +--- nc/views.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/nc/data/importer.py b/nc/data/importer.py index fd461aa0..2e86d44e 100755 --- a/nc/data/importer.py +++ b/nc/data/importer.py @@ -26,9 +26,7 @@ MAGIC_NC_FTP_URL = "ftp://nc.us/" -def run( - url, destination=None, zip_path=None, min_stop_id=None, max_stop_id=None, prime_cache=False -): +def run(url, destination=None, zip_path=None, min_stop_id=None, max_stop_id=None, prime_cache=True): """ Download NC data, extract, convert to CSV, and load into PostgreSQL diff --git a/nc/views.py b/nc/views.py index 153dad09..6d6ef0cb 100644 --- a/nc/views.py +++ b/nc/views.py @@ -12,7 +12,7 @@ from django.db.models import Case, Count, F, Q, Sum, Value, When from django.db.models.functions import ExtractYear from django.utils.decorators import method_decorator -from django.views.decorators.cache import never_cache +from django.views.decorators.cache import cache_page, never_cache from django_filters.rest_framework import DjangoFilterBackend from rest_framework import viewsets from rest_framework.decorators import action @@ -89,6 +89,9 @@ class QueryKeyConstructor(DefaultObjectKeyConstructor): query_cache_key_func = QueryKeyConstructor() +CACHE_TIMEOUT = settings.CACHE_COUNT_TIMEOUT + + def get_date_range(request): # Only filter is from and to values are found and are valid date_precision = "year" @@ -434,6 +437,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): stop_qs = StopSummary.objects.all().annotate(year=ExtractYear("date")) @@ -531,6 +535,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): date_precision, date_range = get_date_range(request) @@ -576,6 +581,7 @@ def get_values(self, df, stop_purpose, years_len): else: return [0] * years_len + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): qs = StopSummary.objects.all() agency_id = int(agency_id) @@ -677,6 +683,7 @@ def get_values(col): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): qs = StopSummary.objects.all() agency_id = int(agency_id) @@ -740,6 +747,7 @@ def get(self, request, agency_id): class AgencyContrabandView(APIView): + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): year = request.GET.get("year", None) @@ -822,6 +830,7 @@ def get(self, request, agency_id): class AgencyContrabandTypesView(APIView): + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): year = request.GET.get("year", None) @@ -943,6 +952,7 @@ def get_values(col): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): year = request.GET.get("year", None) @@ -1094,6 +1104,7 @@ def create_dataset(self, contraband_df, searches_df, stop_purpose): data.append(group) return data + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): year = request.GET.get("year", None) @@ -1162,6 +1173,7 @@ def get(self, request, agency_id): class AgencyContrabandStopGroupByPurposeModalView(APIView): + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): grouped_stop_purpose = request.GET.get("grouped_stop_purpose") contraband_type = request.GET.get("contraband_type") @@ -1271,6 +1283,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): stop_qs = StopSummary.objects.all().annotate(year=ExtractYear("date")) @@ -1383,6 +1396,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): date_precision, date_range = get_date_range(request) @@ -1466,6 +1480,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): stop_qs = StopSummary.objects.all().annotate(year=ExtractYear("date")) search_qs = StopSummary.objects.filter(search_type__isnull=False).annotate( @@ -1592,6 +1607,7 @@ def get_values(race): ], } + @method_decorator(cache_page(CACHE_TIMEOUT)) def get(self, request, agency_id): qs = StopSummary.objects.filter(search_type__isnull=False, engage_force="t").annotate( year=ExtractYear("date") From 26b6c949e8c240db0417d197946a852a44266716 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Wed, 21 Feb 2024 18:03:26 -0500 Subject: [PATCH 05/10] Fix pie charts not loading for traffic stops tab (#272) --- frontend/src/Components/Charts/TrafficStops/TrafficStops.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js index 95eec30b..5853ca74 100644 --- a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js +++ b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js @@ -266,7 +266,7 @@ function TrafficStops(props) { }, []); const buildPercentages = (data, ds) => { - if (!data.length) return [0, 0, 0, 0, 0, 0]; + if (!data.hasOwnProperty(ds)) return [0, 0, 0, 0, 0, 0]; const dsTotal = data[ds].datasets .map((s) => s.data.reduce((a, b) => a + b, 0)) .reduce((a, b) => a + b, 0); From 2f92fa9eafbae1ee80c5881c11c4b3d2d9209228 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Thu, 22 Feb 2024 08:21:20 -0500 Subject: [PATCH 06/10] Check if scalar values aren't nan before adding to json data (#273) --- nc/views.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nc/views.py b/nc/views.py index 6d6ef0cb..50c929e6 100644 --- a/nc/views.py +++ b/nc/views.py @@ -4,6 +4,7 @@ from functools import reduce from operator import concat +import numpy import pandas as pd from dateutil import relativedelta @@ -1439,7 +1440,11 @@ def build_response(self, df, labels): def get_values(race): if race in df: values = [float(df[race][label]) if label in df[race] else 0 for label in labels] - values.insert(0, sum(values) / len(values)) + try: + average = sum(values) / len(values) + except ZeroDivisionError: + average = 0 + values.insert(0, average) return values return [0] * (len(labels) + 1) @@ -1529,8 +1534,9 @@ def get(self, request, agency_id): def get_val(df, column, purpose): if column in df and purpose in df[column]: - return df[column][purpose] - return 0 + val = df[column][purpose] + return float(0) if numpy.isnan(val) else float(val) + return float(0) for col in columns: for k, v in purpose_choices.items(): From eb2ffe73ab0002cbb9f40c65f3e62eb315030011 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Thu, 22 Feb 2024 09:58:18 -0500 Subject: [PATCH 07/10] Add pie chart for traffic stops by stop purpose (#274) --- .../Charts/TrafficStops/TrafficStops.js | 154 +++++++++++++++--- 1 file changed, 134 insertions(+), 20 deletions(-) diff --git a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js index 5853ca74..cb1ed055 100644 --- a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js +++ b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js @@ -113,6 +113,26 @@ function TrafficStops(props) { datasets: [], loading: true, }); + const [groupedStopYear, setGroupedStopYear] = useState(YEARS_DEFAULT); + + const purposeGroupedPieLabels = ['Safety Violation', 'Regulatory and Equipment', 'Other']; + const purposeGroupedPieColors = ['#5F0F40', '#E36414', '#0F4C5C']; + const purposeGroupedPieConfig = { + backgroundColor: purposeGroupedPieColors, + borderColor: purposeGroupedPieColors, + hoverBackgroundColor: purposeGroupedPieColors, + borderWidth: 1, + }; + const [stopPurposeGroupedPieData, setStopPurposeGroupedPieData] = useState({ + labels: purposeGroupedPieLabels, + datasets: [ + { + data: [], + ...purposeGroupedPieConfig, + }, + ], + }); + const [stopsGroupedByPurposeData, setStopsGroupedByPurpose] = useState({ labels: [], safety: { labels: [], datasets: [], loading: true }, @@ -261,11 +281,12 @@ function TrafficStops(props) { .get(url) .then((res) => { setStopPurposeGroups(res.data); + buildStopPurposeGroupedPieData(res.data); }) .catch((err) => console.log(err)); }, []); - const buildPercentages = (data, ds) => { + const buildEthnicPercentages = (data, ds) => { if (!data.hasOwnProperty(ds)) return [0, 0, 0, 0, 0, 0]; const dsTotal = data[ds].datasets .map((s) => s.data.reduce((a, b) => a + b, 0)) @@ -286,9 +307,9 @@ function TrafficStops(props) { .then((res) => { setStopsGroupedByPurpose(res.data); updateStoppedPurposePieChart( - buildPercentages(res.data, 'safety'), - buildPercentages(res.data, 'regulatory'), - buildPercentages(res.data, 'other') + buildEthnicPercentages(res.data, 'safety'), + buildEthnicPercentages(res.data, 'regulatory'), + buildEthnicPercentages(res.data, 'other') ); }) .catch((err) => console.log(err)); @@ -346,6 +367,52 @@ function TrafficStops(props) { setYear(y); }; + const buildStopPurposeGroupedPieData = (ds, stopPurposeYear = null) => { + const getValues = (arr) => { + if (!stopPurposeYear) { + return arr.reduce((a, b) => a + b, 0); + } + // Reverse to match dropdown descending years + return arr.toReversed()[stopPurposeYear - 1] || 0; + }; + + const data = []; + if (ds) { + const safety = getValues(ds.datasets[0].data); + const regulatory = getValues(ds.datasets[1].data); + const other = getValues(ds.datasets[2].data); + const total = safety + regulatory + other; + + const normalize = (n) => ((n / total) * 100).toFixed(4); + + if (total > 0) { + data.push(normalize(safety)); + data.push(normalize(regulatory)); + data.push(normalize(other)); + } + } + setStopPurposeGroupedPieData({ + labels: purposeGroupedPieLabels, + datasets: [ + { + data, + ...purposeGroupedPieConfig, + }, + ], + }); + }; + + const handleGroupedStopPurposeYearSelect = (y, i) => { + if (y === groupedStopYear) return; + + setGroupedStopYear(y); + if (y === YEARS_DEFAULT) { + // eslint-disable-next-line no-param-reassign + i = null; + } + buildStopPurposeGroupedPieData(stopPurposeGroupsData, i); + }; + // Handle stop purpose dropdown state const handleStopPurposeSelect = (p, i) => { if (p === purpose) return; @@ -475,7 +542,7 @@ function TrafficStops(props) { setChecked(nextChecked); }; - const buildPercentagesForYear = (data, ds, idx = null) => { + const buildEthnicPercentagesForYear = (data, ds, idx = null) => { const dsTotal = data[ds].datasets.map((s) => s.data[idx]).reduce((a, b) => a + b, 0); return data[ds].datasets.map((s) => ((s.data[idx] / dsTotal) * 100 || 0).toFixed(2)); }; @@ -486,14 +553,14 @@ function TrafficStops(props) { const idxForYear = stopsGroupedByPurposeData.labels.length - idx; updateStoppedPurposePieChart( selectedYear === YEARS_DEFAULT - ? buildPercentages(stopsGroupedByPurposeData, 'safety') - : buildPercentagesForYear(stopsGroupedByPurposeData, 'safety', idxForYear), + ? buildEthnicPercentages(stopsGroupedByPurposeData, 'safety') + : buildEthnicPercentagesForYear(stopsGroupedByPurposeData, 'safety', idxForYear), selectedYear === YEARS_DEFAULT - ? buildPercentages(stopsGroupedByPurposeData, 'regulatory') - : buildPercentagesForYear(stopsGroupedByPurposeData, 'regulatory', idxForYear), + ? buildEthnicPercentages(stopsGroupedByPurposeData, 'regulatory') + : buildEthnicPercentagesForYear(stopsGroupedByPurposeData, 'regulatory', idxForYear), selectedYear === YEARS_DEFAULT - ? buildPercentages(stopsGroupedByPurposeData, 'other') - : buildPercentagesForYear(stopsGroupedByPurposeData, 'other', idxForYear) + ? buildEthnicPercentages(stopsGroupedByPurposeData, 'other') + : buildEthnicPercentagesForYear(stopsGroupedByPurposeData, 'other', idxForYear) ); }; @@ -551,15 +618,27 @@ function TrafficStops(props) { subject = `Officer ${officerId}`; } return `Traffic Stops By Percentage for ${subject} ${ - year === YEARS_DEFAULT ? `since ${stopsChartState.yearRange.reverse()[0]}` : `in ${year}` + year === YEARS_DEFAULT ? `since ${stopsChartState.yearRange.toReversed()[0]}` : `in ${year}` }`; }; - const getPieChartModalSubHeading = (title) => { - const yearSelected = year && year !== 'All' ? ` in ${year}` : ''; + const getPieChartModalSubHeading = (title, yr = null) => { + const yearSelected = yr && yr !== 'All' ? ` in ${yr}` : ''; return `${title} ${subjectObserving()}${yearSelected}.`; }; + const stopPurposeGroupPieChartTitle = () => { + let subject = stopsChartState.data[AGENCY_DETAILS].name; + if (officerId) { + subject = `Officer ${officerId}`; + } + return `Traffic Stops By Stop Purpose for ${subject} ${ + groupedStopYear === YEARS_DEFAULT + ? `since ${stopsGroupedByPurposeData.labels[0]}` + : `in ${groupedStopYear}` + }`; + }; + const getPieChartModalHeading = (stopPurpose) => { let subject = stopsChartState.data[AGENCY_DETAILS].name; if (officerId) { @@ -608,6 +687,14 @@ function TrafficStops(props) { return `Traffic Stops by Percentage for ${subject} since ${stopsByPercentageData.labels[0]}`; }; + const stopPurposeGroupedPieYears = () => { + if (stopPurposeGroupsData.labels) { + const years = [...stopPurposeGroupsData.labels].toReversed(); + return [YEARS_DEFAULT].concat(years); + } + return [YEARS_DEFAULT]; + }; + return ( {/* Traffic Stops by Percentage */} @@ -650,7 +737,8 @@ function TrafficStops(props) { modalConfig={{ tableHeader: 'Traffic Stops By Percentage', tableSubheader: getPieChartModalSubHeading( - 'Shows the race/ethnic composition of drivers stopped' + 'Shows the race/ethnic composition of drivers stopped', + year ), agencyName: stopsChartState.data[AGENCY_DETAILS].name, chartTitle: pieChartTitle(), @@ -662,7 +750,7 @@ function TrafficStops(props) { label="Year" value={year} onChange={handleYearSelect} - options={[YEARS_DEFAULT].concat(stopsChartState.yearRange)} + options={[YEARS_DEFAULT].concat(stopsByPercentageData.labels.toReversed())} /> @@ -738,8 +826,8 @@ function TrafficStops(props) { isOpen={stopPurposeModalData.isOpen} closeModal={() => setStopPurposeModalData((state) => ({ ...state, isOpen: false }))} /> - - + + - - + + + + + + + + + + Date: Fri, 23 Feb 2024 11:16:34 -0500 Subject: [PATCH 08/10] Update prime cache to skip officers for now and include statewide data (#275) --- nc/management/commands/prime_cache.py | 2 +- nc/prime_cache.py | 19 +++++++++++++++---- nc/tests/test_prime_cache.py | 3 +++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/nc/management/commands/prime_cache.py b/nc/management/commands/prime_cache.py index 152ca83a..a479ea64 100755 --- a/nc/management/commands/prime_cache.py +++ b/nc/management/commands/prime_cache.py @@ -14,7 +14,7 @@ def add_arguments(self, parser): ) parser.add_argument("--clear-cache", "-c", action="store_true", default=False) parser.add_argument("--skip-agencies", action="store_true", default=False) - parser.add_argument("--skip-officers", action="store_true", default=False) + parser.add_argument("--skip-officers", action="store_true", default=True) parser.add_argument( "--officer-cutoff-count", type=int, diff --git a/nc/prime_cache.py b/nc/prime_cache.py index 9b0c4f74..06008bd6 100755 --- a/nc/prime_cache.py +++ b/nc/prime_cache.py @@ -123,7 +123,7 @@ def get_endpoints(self): def prime(self): logger.info(f"{self} starting") - self.count = self.get_queryset().count() + self.count = len(self.get_queryset()) logger.info(f"{self} priming {self.count:,} objects") for endpoints in self.get_endpoints(): for endpoint in endpoints: @@ -140,13 +140,24 @@ def __repr__(self): class AgencyStopsPrimer(CachePrimer): def get_queryset(self): - return ( + qs = list( Stop.objects.no_cache() .annotate(agency_name=F("agency_description")) .values("agency_name", "agency_id") .annotate(num_stops=Count("stop_id")) .order_by("-num_stops") ) + # Manually insert the statewide to force the caching since a + # stop instance won't directly be associated with the statewide agency id. + qs.insert( + 0, + { + "agency_name": "North Carolina State", + "agency_id": -1, + "num_stops": Stop.objects.count(), + }, + ) + return qs def get_urls(self, row): urls = [] @@ -177,7 +188,7 @@ def run( cutoff_duration_secs=None, clear_cache=False, skip_agencies=False, - skip_officers=False, + skip_officers=True, officer_cutoff_count=None, ): """ @@ -216,6 +227,6 @@ def run( if not skip_officers: OfficerStopsPrimer( cutoff_secs=0, cutoff_count=officer_cutoff_count - ).prime() # cache all officer endpoins for now + ).prime() # cache all officer endpoints for now logger.info("Complete") diff --git a/nc/tests/test_prime_cache.py b/nc/tests/test_prime_cache.py index 5ba04a2d..0d1aec8f 100755 --- a/nc/tests/test_prime_cache.py +++ b/nc/tests/test_prime_cache.py @@ -1,5 +1,6 @@ from django.test import TestCase +from nc.models import Agency from nc.prime_cache import run from nc.tests import factories @@ -13,6 +14,8 @@ class PrimeCacheTests(TestCase): databases = "__all__" def test_prime_cache(self): + factories.AgencyFactory(id=-1) # Statewide data + factories.ContrabandFactory() factories.ContrabandFactory() factories.ContrabandFactory() From a04523cf33e56b16eccd07786e2b151f081ff770 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Fri, 23 Feb 2024 11:42:54 -0500 Subject: [PATCH 09/10] Fix contraband dropdown cutoff when comparing departments (#276) --- frontend/src/Components/Charts/Contraband/Contraband.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/Components/Charts/Contraband/Contraband.js b/frontend/src/Components/Charts/Contraband/Contraband.js index 4637a4f0..5da52ee9 100644 --- a/frontend/src/Components/Charts/Contraband/Contraband.js +++ b/frontend/src/Components/Charts/Contraband/Contraband.js @@ -551,7 +551,7 @@ function Contraband(props) { value={year} onChange={handleYearSelect} options={[YEARS_DEFAULT].concat(chartState.yearRange)} - dropUp={!!showCompare} + dropDown /> From 376679341e55e536121e3445194cc79ca9a3ed82 Mon Sep 17 00:00:00 2001 From: Aristotel Fani Date: Mon, 26 Feb 2024 09:58:53 -0500 Subject: [PATCH 10/10] Refactor init data for graphs that should reload when user chooses year from dropdown, etc. (#277) --- .../Charts/Contraband/Contraband.js | 28 +++++++++++++------ .../Charts/SearchRate/SearchRate.js | 5 +++- .../Components/Charts/Searches/Searches.js | 6 ++-- .../Charts/TrafficStops/TrafficStops.js | 6 ++-- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/frontend/src/Components/Charts/Contraband/Contraband.js b/frontend/src/Components/Charts/Contraband/Contraband.js index 5da52ee9..5a0d1c26 100644 --- a/frontend/src/Components/Charts/Contraband/Contraband.js +++ b/frontend/src/Components/Charts/Contraband/Contraband.js @@ -47,28 +47,35 @@ function Contraband(props) { const renderMetaTags = useMetaTags(); const [renderTableModal] = useTableModal(); - const [contrabandData, setContrabandData] = useState({ + + const initContrabandData = { labels: [], datasets: [], isModalOpen: false, tableData: [], csvData: [], loading: true, - }); - const [contrabandTypesData, setContrabandTypesData] = useState({ + }; + const [contrabandData, setContrabandData] = useState(initContrabandData); + + const initContrabandTypesData = { labels: [], datasets: [], isModalOpen: false, tableData: [], csvData: [], loading: true, - }); + }; + const [contrabandTypesData, setContrabandTypesData] = useState(initContrabandTypesData); - const [contrabandStopPurposeData, setContrabandStopPurposeData] = useState({ + const initContrabandStopPurposeData = { labels: [], datasets: [], loading: true, - }); + }; + const [contrabandStopPurposeData, setContrabandStopPurposeData] = useState( + initContrabandStopPurposeData + ); const [contrabandStopPurposeModalData, setContrabandStopPurposeModalData] = useState({ modalData: {}, isOpen: false, @@ -87,7 +94,7 @@ function Contraband(props) { loading: true, }); - const initialContrabandGroupedData = [ + const initContrabandGroupedStopPurposeData = [ { labels: [], datasets: [], @@ -105,7 +112,7 @@ function Contraband(props) { }, ]; const [contrabandGroupedStopPurposeData, setContrabandGroupedStopPurposeData] = useState( - initialContrabandGroupedData + initContrabandGroupedStopPurposeData ); const [shouldRedrawContrabandGraphs, setShouldReDrawContrabandGraphs] = useState(true); const [contrabandTypes, setContrabandTypes] = useState(() => @@ -145,7 +152,10 @@ function Contraband(props) { const handleYearSelect = (y) => { if (y === year) return; setYear(y); - setContrabandGroupedStopPurposeData(initialContrabandGroupedData); + setContrabandData(initContrabandData); + setContrabandTypesData(initContrabandTypesData); + setContrabandStopPurposeData(initContrabandStopPurposeData); + setContrabandGroupedStopPurposeData(initContrabandGroupedStopPurposeData); fetchHitRateByStopPurpose(y); }; diff --git a/frontend/src/Components/Charts/SearchRate/SearchRate.js b/frontend/src/Components/Charts/SearchRate/SearchRate.js index 28772c84..00c203e3 100644 --- a/frontend/src/Components/Charts/SearchRate/SearchRate.js +++ b/frontend/src/Components/Charts/SearchRate/SearchRate.js @@ -27,12 +27,15 @@ function SearchRate(props) { const [chartState] = useDataset(agencyId, LIKELIHOOD_OF_SEARCH); const [year, setYear] = useState(YEARS_DEFAULT); - const [searchRateData, setSearchRateData] = useState({ labels: [], datasets: [], loading: true }); + + const initData = { labels: [], datasets: [], loading: true }; + const [searchRateData, setSearchRateData] = useState(initData); const renderMetaTags = useMetaTags(); const [renderTableModal, { openModal }] = useTableModal(); useEffect(() => { + setSearchRateData(initData); const params = []; if (year && year !== 'All') { params.push({ param: 'year', val: year }); diff --git a/frontend/src/Components/Charts/Searches/Searches.js b/frontend/src/Components/Charts/Searches/Searches.js index 65fc5540..877bf397 100644 --- a/frontend/src/Components/Charts/Searches/Searches.js +++ b/frontend/src/Components/Charts/Searches/Searches.js @@ -52,11 +52,12 @@ function Searches(props) { loading: true, }); - const [searchCountData, setSearchCountData] = useState({ + const initCountData = { labels: [], datasets: [], loading: true, - }); + }; + const [searchCountData, setSearchCountData] = useState(initCountData); const [searchCountType, setSearchCountType] = useState(0); const renderMetaTags = useMetaTags(); const [renderTableModal, { openModal }] = useTableModal(); @@ -81,6 +82,7 @@ function Searches(props) { // Build Searches By Count useEffect(() => { + setSearchCountData(initCountData); const params = []; if (searchType !== 0) { params.push({ param: 'search_type', val: searchCountType }); diff --git a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js index cb1ed055..69e57578 100644 --- a/frontend/src/Components/Charts/TrafficStops/TrafficStops.js +++ b/frontend/src/Components/Charts/TrafficStops/TrafficStops.js @@ -223,17 +223,19 @@ function TrafficStops(props) { null, null, ]); - const [trafficStopsByCount, setTrafficStopsByCount] = useState({ + const initStopsByCount = { labels: [], datasets: [], loading: true, - }); + }; + const [trafficStopsByCount, setTrafficStopsByCount] = useState(initStopsByCount); const createDateForRange = (yr) => Number.isInteger(yr) ? new Date(`${yr}-01-01`) : new Date(yr); // Build Stops By Count useEffect(() => { + setTrafficStopsByCount(initStopsByCount); const params = []; if (trafficStopsByCountRange !== null) { const _from = `${trafficStopsByCountRange.from.year}-${trafficStopsByCountRange.from.month