Skip to content

Commit f3d48c8

Browse files
authored
Merge pull request #1033 from mapswipe/dev
Release - 2025 May
2 parents b2544d6 + 408fb60 commit f3d48c8

File tree

5 files changed

+179
-7
lines changed

5 files changed

+179
-7
lines changed

django/apps/aggregated/management/commands/update_aggregated_data.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
5656
-- FOOTPRINT: Not calculated right now
5757
WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
58+
WHEN P.project_type = {Project.Type.STREET.value} THEN 65
5859
ELSE 1
5960
END
6061
) * COUNT(*) as time_spent_max_allowed
@@ -110,6 +111,7 @@
110111
WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
111112
-- FOOTPRINT: Not calculated right now
112113
WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
114+
WHEN P.project_type = {Project.Type.STREET.value} THEN 65
113115
ELSE 1
114116
END
115117
) * COUNT(*) as time_spent_max_allowed
121 KB
Loading

docs/source/diagrams.md

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,29 +9,56 @@ The Diagrams are drawn using [draw.io](https://.wwww.draw.io). You can download
99
---
1010

1111
**Deployment Diagram:**
12-
![Deployment Diagram](/_static/img/deployment_diagram.png)
12+
![Deployment Diagram](_static/img/deployment_diagram.png)
1313

1414
---
1515

1616
**Proposed Data Structure Project Type 1 - Firebase:**
17-
![Data Structure - Firebase](/_static/img/data_structure-firebase-1.svg)
17+
![Data Structure - Firebase](_static/img/data_structure-firebase-1.svg)
1818

1919
---
2020

2121
**Proposed Data Structure Project Type 2 - Firebase:**
22-
![Data Structure - Firebase](/_static/img/data_structure-firebase-2.svg)
22+
![Data Structure - Firebase](_static/img/data_structure-firebase-2.svg)
2323

2424
---
2525

2626
**Database Scheme - Postgres:**
27-
![Database Schema - Postgres](/_static/img/database_schema-postgres.png)
27+
![Database Schema - Postgres](_static/img/database_schema-postgres.png)
2828

2929
---
3030

3131
**Entity Relationship Diagram - Postgres:**
32-
![Entity Relationship Diagram- Postgres](/_static/img/entity_relationship_diagram-postgres.png)
32+
![Entity Relationship Diagram- Postgres](_static/img/entity_relationship_diagram-postgres.png)
3333

3434
---
3535

3636
**Database Schema - Analytics:**
37-
![Database Schema - Analytics](/_static/img/database_schema-analytics.png)
37+
![Database Schema - Analytics](_static/img/database_schema-analytics.png)
38+
39+
---
40+
41+
**Mapping Sessions - Time Calculation**
42+
43+
The diagram below is a visual representation of how time is calculated in MapSwipe.
44+
45+
Step 1: User Mapping Session **sends data** to Firebase
46+
- When a user completes a mapping session in the mobile/web app, the session payload (including start/end timestamps, user ID, session metadata, etc.) is sent in real time to Firebase.
47+
48+
Step 2: Cron job **fetches data** from the firebase
49+
- Every 3 minutes, a cron job syncs data for any new session records and pulls them into the backend.
50+
51+
Step 3: Cron job **saves raw data** to Postgres database
52+
- The cron job sends new session data to the Postgres database.
53+
54+
Step 4: Cron job **reads raw data** from Postgres database
55+
- Another cron job reads the raw data from Postgres database.
56+
57+
Step 5: Cron job **saves aggregates** to Postgres database
58+
- The cron job aggregates previous 24 hours data (end date - start date), sends back, and saves processed aggregated data to the Postgres database.
59+
60+
Step 6: Community dashboard **queries aggregate data** from Postgres database
61+
- The Community dashboard pulls the processed data from the Postgres database and updates the dashbaord with up-to-date stats.
62+
63+
64+
![MapSwipe Time Calculation](_static/img/mapswipe-time-calculation.png)

manager-dashboard/app/views/NewProject/utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,7 @@ async function fetchAoiFromHotTaskingManager(projectId: number | string): (
681681
let response;
682682
try {
683683
response = await fetch(
684-
`https://tasking-manager-tm4-production-api.hotosm.org/api/v2/projects/${projectId}/queries/aoi/?as_file=false`,
684+
`https://tasking-manager-production-api.hotosm.org/api/v2/projects/${projectId}/queries/aoi/?as_file=false`,
685685
);
686686
} catch {
687687
return {
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import argparse
2+
import os
3+
import warnings
4+
5+
import geopandas as gpd
6+
import pandas as pd
7+
import rasterio
8+
import requests
9+
from exactextract import exact_extract
10+
from tqdm import tqdm
11+
12+
warnings.filterwarnings("ignore")
13+
14+
15+
def project_list(id_file):
16+
"""Reads Mapswipe project IDs from the user input file"""
17+
18+
with open(id_file, "r") as file:
19+
ids = file.read().strip()
20+
21+
project_list = ids.split(",")
22+
project_list = [id.strip() for id in project_list]
23+
24+
return project_list
25+
26+
27+
def population_raster_download():
28+
"""Downloads 1km resolution global population raster for 2020 from WorldPop to the current working directory."""
29+
30+
url = "https://data.worldpop.org/GIS/Population/Global_2000_2020/2020/0_Mosaicked/ppp_2020_1km_Aggregated.tif"
31+
32+
output_file = "ppp_2020_1km_Aggregated.tif"
33+
34+
output_file_path = os.path.join(os.getcwd(), output_file)
35+
36+
if os.path.exists(output_file_path):
37+
38+
print("Population raster already exists. Moving to next steps......")
39+
return output_file_path
40+
41+
else:
42+
43+
response = requests.get(url, stream=True)
44+
size = int(response.headers.get("content-length", 0))
45+
block_size = 1024
46+
try:
47+
with open(output_file, "wb") as file, tqdm(
48+
desc="Downloading population raster",
49+
total=size,
50+
unit="B",
51+
unit_scale=True,
52+
unit_divisor=1024,
53+
) as bar:
54+
for chunk in response.iter_content(block_size):
55+
if chunk:
56+
file.write(chunk)
57+
bar.update(len(chunk))
58+
59+
print("Download complete:", output_file_path)
60+
return output_file_path
61+
62+
except requests.RequestException as e:
63+
print(f"Error downloading data: {e}")
64+
65+
66+
def population_count(list, dir, raster):
67+
"""Gets boundary data for projects from Mapswipe API and calculates zonal statistics
68+
with global population raster and individual project boundaries."""
69+
70+
dict = {}
71+
worldpop = rasterio.open(raster)
72+
73+
for id in list:
74+
url = f"https://apps.mapswipe.org/api/project_geometries/project_geom_{id}.geojson"
75+
response = requests.get(url)
76+
77+
try:
78+
geojson = response.json()
79+
for feature in geojson["features"]:
80+
geometry = feature.get("geometry", {})
81+
if "coordinates" in geometry:
82+
if geometry["type"] == "Polygon":
83+
geometry["coordinates"] = [
84+
[[coord[0], coord[1]] for coord in polygon]
85+
for polygon in geometry["coordinates"]
86+
]
87+
elif geometry["type"] == "MultiPolygon":
88+
geometry["coordinates"] = [
89+
[
90+
[[coord[0], coord[1]] for coord in polygon]
91+
for polygon in multipolygon
92+
]
93+
for multipolygon in geometry["coordinates"]
94+
]
95+
gdf = gpd.GeoDataFrame.from_features(geojson["features"])
96+
gdf.set_crs("EPSG:4326", inplace=True)
97+
no_of_people = exact_extract(worldpop, gdf, "sum")
98+
no_of_people = round(no_of_people[0]["properties"]["sum"])
99+
100+
dict[id] = no_of_people
101+
102+
except requests.RequestException as e:
103+
print(f"Error in retrieval of project boundary from Mapswipe: {e}")
104+
105+
df = pd.DataFrame(
106+
dict.items(), columns=["Project_IDs", "Number of people impacted"]
107+
)
108+
109+
df["Project_IDs"] = "https://mapswipe.org/en/projects/" + df["Project_IDs"]
110+
111+
df.to_csv(f"{dir}/projects_population.csv")
112+
113+
print(f"CSV file successfully created at {dir}/number_of_people_impacted.csv")
114+
115+
116+
if __name__ == "__main__":
117+
"""Generates population stats for individual Mapswipe projects"""
118+
parser = argparse.ArgumentParser()
119+
parser.add_argument(
120+
"-t",
121+
"--text_file",
122+
help=(
123+
"Path to the text file containing project IDs from Mapswipe. The file should contain IDs in this manner: "
124+
"-O8kulfxD4zRYQ2T1aXf, -O8kyOCreRGklW15n8RU, -O8kzSy9105axIPOAJjO, -OAwWv9rnJqPXTpWxO8-, "
125+
"-OB-tettI2np7t3Gpu-k"
126+
),
127+
type=str,
128+
required=True,
129+
)
130+
parser.add_argument(
131+
"-o",
132+
"--output_directory",
133+
help="Path to the directory to store the output",
134+
type=str,
135+
required=True,
136+
)
137+
args = parser.parse_args()
138+
139+
population_count(
140+
project_list(args.text_file),
141+
args.output_directory,
142+
population_raster_download(),
143+
)

0 commit comments

Comments
 (0)