From d3d92134e03ab4d1d92ec31f09fd5aadfa9db13d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 8 May 2025 14:20:50 +0200 Subject: [PATCH 01/35] feat: osm auth for web client --- docker-compose.yaml | 10 ++- example.env | 5 ++ firebase/functions/src/index.ts | 10 ++- firebase/functions/src/osm_auth.ts | 128 ++++++++++++++++++++++++++++- 4 files changed, 150 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index d465a704c..b1afabb16 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -230,15 +230,23 @@ services: OSM_OAUTH_API_URL: '${OSM_OAUTH_API_URL}' OSM_OAUTH_CLIENT_ID: '${OSM_OAUTH_CLIENT_ID}' OSM_OAUTH_CLIENT_SECRET: '${OSM_OAUTH_CLIENT_SECRET}' + OSM_OAUTH_REDIRECT_URI_WEB: '${OSM_OAUTH_REDIRECT_URI_WEB}' + OSM_OAUTH_APP_LOGIN_LINK_WEB: '${OSM_APP_LOGIN_LINK_WEB}' + OSM_OAUTH_CLIENT_ID_WEB: '${OSM_OAUTH_CLIENT_ID_WEB}' + OSM_OAUTH_CLIENT_SECRET_WEB: '${OSM_OAUTH_CLIENT_SECRET_WEB}' command: >- sh -c "firebase use $FIREBASE_DB && firebase target:apply hosting auth \"$FIREBASE_AUTH_SITE\" && firebase functions:config:set osm.redirect_uri=\"$OSM_OAUTH_REDIRECT_URI\" + osm.redirect_uri_web=\"$OSM_OAUTH_REDIRECT_URI_WEB\" osm.app_login_link=\"$OSM_OAUTH_APP_LOGIN_LINK\" + osm.app_login_link_web=\"$OSM_OAUTH_APP_LOGIN_LINK_WEB\" osm.api_url=\"$OSM_OAUTH_API_URL\" osm.client_id=\"$OSM_OAUTH_CLIENT_ID\" - osm.client_secret=\"$OSM_OAUTH_CLIENT_SECRET\" && + osm.client_id_web=\"$OSM_OAUTH_CLIENT_ID_WEB\" + osm.client_secret=\"$OSM_OAUTH_CLIENT_SECRET\" + osm.client_secret_web=\"$OSM_OAUTH_CLIENT_SECRET_WEB\" && firebase deploy --token $FIREBASE_TOKEN --only functions,hosting,database" django: diff --git a/example.env b/example.env index 2fa92ae33..3b5024245 100644 --- a/example.env +++ b/example.env @@ -38,10 +38,15 @@ OSMCHA_API_KEY= # OSM OAuth Configuration OSM_OAUTH_REDIRECT_URI= +OSM_OAUTH_REDIRECT_URI_WEB= OSM_OAUTH_API_URL= OSM_OAUTH_CLIENT_ID= +OSM_OAUTH_CLIENT_ID_WEB= OSM_OAUTH_CLIENT_SECRET= +OSM_OAUTH_CLIENT_SECRET_WEB= OSM_APP_LOGIN_LINK= +OSM_APP_LOGIN_LINK_WEB= + # DJANGO For more info look at django/mapswipe/settings.py::L22 DJANGO_SECRET_KEY= diff --git a/firebase/functions/src/index.ts b/firebase/functions/src/index.ts index 793ed9c71..6bb56e98c 100644 --- a/firebase/functions/src/index.ts +++ b/firebase/functions/src/index.ts @@ -8,7 +8,7 @@ admin.initializeApp(); // all functions are bundled together. It's less than ideal, but it does not // seem possible to split them using the split system for multiple sites from // https://firebase.google.com/docs/hosting/multisites -import {redirect, token} from './osm_auth'; +import {redirect, token, redirect_web, token_web} from './osm_auth'; import { formatProjectTopic, formatUserName } from './utils'; exports.osmAuth = {}; @@ -23,6 +23,14 @@ exports.osmAuth.token = functions.https.onRequest((req, res) => { token(req, res, admin); }); +exports.osmAuth.redirect_web = functions.https.onRequest((req, res) => { + redirect_web(req, res); +}); + +exports.osmAuth.token_web = functions.https.onRequest((req, res) => { + token_web(req, res, admin); +}); + /* Log the userIds of all users who finished a group to /v2/userGroups/{projectId}/{groupId}/. Gets triggered when new results of a group are written to the database. diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index d187b4e4f..5b3a84605 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -1,4 +1,4 @@ -// Firebase cloud functions to allow authentication with OpenStreet Map +// Firebase cloud functions to allow authentication with OpenStreetMap // // There are really 2 functions, which must be publicly accessible via // an https endpoint. They can be hosted on firebase under a domain like @@ -20,8 +20,10 @@ import axios from 'axios'; // will get a cryptic error about the server not being able to continue // TODO: adjust the prefix based on which deployment is done (prod/dev) const OAUTH_REDIRECT_URI = functions.config().osm?.redirect_uri; +const OAUTH_REDIRECT_URI_WEB = functions.config().osm?.redirect_web; const APP_OSM_LOGIN_DEEPLINK = functions.config().osm?.app_login_link; +const APP_OSM_LOGIN_DEEPLINK_WEB = functions.config().osm?.app_login_link_web; // the scope is taken from https://wiki.openstreetmap.org/wiki/OAuth#OAuth_2.0 // at least one seems to be required for the auth workflow to complete. @@ -51,6 +53,21 @@ function osmOAuth2Client() { return simpleOAuth2.create(credentials); } +function osmOAuth2ClientWeb() { + const credentials = { + client: { + id: functions.config().osm?.client_id_web, + secret: functions.config().osm?.client_secret_web, + }, + auth: { + tokenHost: OSM_API_URL, + tokenPath: '/oauth2/token', + authorizePath: '/oauth2/authorize', + }, + }; + return simpleOAuth2.create(credentials); +} + /** * Redirects the User to the OSM authentication consent screen. * Also the '__session' cookie is set for later state verification. @@ -84,6 +101,32 @@ export const redirect = (req: any, res: any) => { }); }; +export const redirect_web = (req: any, res: any) => { + const oauth2 = osmOAuth2ClientWeb(); + + cookieParser()(req, res, () => { + const state = + req.cookies.state || crypto.randomBytes(20).toString('hex'); + functions.logger.log('Setting verification state:', state); + // the cookie MUST be called __session for hosted functions not to + // strip it from incoming requests + // (https://firebase.google.com/docs/hosting/manage-cache#using_cookies) + res.cookie('__session', state.toString(), { + // cookie is valid for 1 hour + maxAge: 3600000, + secure: true, + httpOnly: true, + }); + const redirectUri = oauth2.authorizationCode.authorizeURL({ + redirect_uri: OAUTH_REDIRECT_URI_WEB, + scope: OAUTH_SCOPES, + state: state, + }); + functions.logger.log('Redirecting to:', redirectUri); + res.redirect(redirectUri); + }); +}; + /** * The OSM OAuth endpoing does not give us any info about the user, * so we need to get the user profile from this endpoint @@ -189,6 +232,89 @@ export const token = async (req: any, res: any, admin: any) => { } }; + +export const token_web = async (req: any, res: any, admin: any) => { + const oauth2 = osmOAuth2ClientWeb(); + + try { + return cookieParser()(req, res, async () => { + functions.logger.log( + 'Received verification state:', + req.cookies.__session, + ); + functions.logger.log('Received state:', req.query.state); + // FIXME: For security, we need to check the cookie that was set + // in the /redirect_web function on the user's browser. + // However, there seems to be a bug in firebase around this. + // https://github.com/firebase/firebase-functions/issues/544 + // and linked SO question + // firebase docs mention the need for a cookie middleware, but there + // is no info about it :( + // cross site cookies don't seem to be the issue + // WE just need to make sure the domain set on the cookies is right + if (!req.cookies.__session) { + throw new Error('State cookie not set or expired. Maybe you took too long to authorize. Please try again.'); + } else if (req.cookies.__session !== req.query.state) { + throw new Error('State validation failed'); + } + functions.logger.log('Received auth code:', req.query.code); + let results; + + try { + // TODO: try adding auth data to request headers if + // this doesn't work + results = await oauth2.authorizationCode.getToken({ + code: req.query.code, + redirect_uri: OAUTH_REDIRECT_URI, + scope: OAUTH_SCOPES, + state: req.query.state, + }); + } catch (error: any) { + functions.logger.log('Auth token error', error, error.data.res.req); + } + // why is token called twice? + functions.logger.log( + 'Auth code exchange result received:', + results, + ); + + // We have an OSM access token and the user identity now. + const accessToken = results && results.access_token; + if (accessToken === undefined) { + throw new Error( + 'Could not get an access token from OpenStreetMap', + ); + } + // get the OSM user id and display_name + const { id, display_name } = await getOSMProfile(accessToken); + functions.logger.log('osmuser:', id, display_name); + if (id === undefined) { + // this should not happen, but help guard against creating + // invalid accounts + throw new Error('Could not obtain an account id from OSM'); + } + + // Create a Firebase account and get the Custom Auth Token. + const firebaseToken = await createFirebaseAccount( + admin, + id, + display_name, + accessToken, + ); + // build a deep link so we can send the token back to the app + // from the browser + const signinUrl = `${APP_OSM_LOGIN_DEEPLINK_WEB}?token=${firebaseToken}`; + functions.logger.log('redirecting user to', signinUrl); + res.redirect(signinUrl); + }); + } catch (error: any) { + // FIXME: this should show up in the user's browser as a bit of text + // We should figure out the various error codes available and feed them + // back into the app to allow the user to take action + return res.json({ error: error.toString() }); + } +}; + /** * Creates a Firebase account with the given user profile and returns a custom * auth token allowing the user to sign in to this account on the app. From 2885ddc3696cfefd7ed61d6ed67e2790f98550cf Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 8 May 2025 14:32:45 +0200 Subject: [PATCH 02/35] fix: use web redirect uri in token_web --- firebase/functions/src/osm_auth.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 5b3a84605..80c157774 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -20,7 +20,7 @@ import axios from 'axios'; // will get a cryptic error about the server not being able to continue // TODO: adjust the prefix based on which deployment is done (prod/dev) const OAUTH_REDIRECT_URI = functions.config().osm?.redirect_uri; -const OAUTH_REDIRECT_URI_WEB = functions.config().osm?.redirect_web; +const OAUTH_REDIRECT_URI_WEB = functions.config().osm?.redirect_uri_web; const APP_OSM_LOGIN_DEEPLINK = functions.config().osm?.app_login_link; const APP_OSM_LOGIN_DEEPLINK_WEB = functions.config().osm?.app_login_link_web; @@ -265,7 +265,7 @@ export const token_web = async (req: any, res: any, admin: any) => { // this doesn't work results = await oauth2.authorizationCode.getToken({ code: req.query.code, - redirect_uri: OAUTH_REDIRECT_URI, + redirect_uri: OAUTH_REDIRECT_URI_WEB, scope: OAUTH_SCOPES, state: req.query.state, }); From f5f0c61f5490423130677df3e7280db332eb6ff9 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 8 May 2025 15:14:05 +0200 Subject: [PATCH 03/35] fix: adapt firebase.json for web app osm login --- firebase/firebase.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/firebase/firebase.json b/firebase/firebase.json index 4c56a3044..469e96e11 100644 --- a/firebase/firebase.json +++ b/firebase/firebase.json @@ -20,6 +20,14 @@ { "source": "/token", "function": "osmAuth-token" + }, + { + "source": "/redirect_web", + "function": "osmAuth-redirect_web" + }, + { + "source": "/token_web", + "function": "osmAuth-token_web" } ] }, From c25951918b20335223c2eabe39a4be4d3c4156af Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 14 May 2025 17:51:31 +0200 Subject: [PATCH 04/35] fix: osm oauth app login link env variable --- docker-compose.yaml | 2 +- example.env | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b1afabb16..f015a71a4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -231,7 +231,7 @@ services: OSM_OAUTH_CLIENT_ID: '${OSM_OAUTH_CLIENT_ID}' OSM_OAUTH_CLIENT_SECRET: '${OSM_OAUTH_CLIENT_SECRET}' OSM_OAUTH_REDIRECT_URI_WEB: '${OSM_OAUTH_REDIRECT_URI_WEB}' - OSM_OAUTH_APP_LOGIN_LINK_WEB: '${OSM_APP_LOGIN_LINK_WEB}' + OSM_OAUTH_APP_LOGIN_LINK_WEB: '${OSM_OAUTH_APP_LOGIN_LINK_WEB}' OSM_OAUTH_CLIENT_ID_WEB: '${OSM_OAUTH_CLIENT_ID_WEB}' OSM_OAUTH_CLIENT_SECRET_WEB: '${OSM_OAUTH_CLIENT_SECRET_WEB}' command: >- diff --git a/example.env b/example.env index 3b5024245..a4ee41436 100644 --- a/example.env +++ b/example.env @@ -44,8 +44,8 @@ OSM_OAUTH_CLIENT_ID= OSM_OAUTH_CLIENT_ID_WEB= OSM_OAUTH_CLIENT_SECRET= OSM_OAUTH_CLIENT_SECRET_WEB= -OSM_APP_LOGIN_LINK= -OSM_APP_LOGIN_LINK_WEB= +OSM_OAUTH_APP_LOGIN_LINK= +OSM_OAUTH_APP_LOGIN_LINK_WEB= # DJANGO For more info look at django/mapswipe/settings.py::L22 From b006d8b51d86d19248060532d6fdea0ad1b9b1e0 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 15 May 2025 09:20:46 +0200 Subject: [PATCH 05/35] fix: avoid underscore in fb function names --- firebase/firebase.json | 8 ++++---- firebase/functions/src/index.ts | 10 +++++----- firebase/functions/src/osm_auth.ts | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/firebase/firebase.json b/firebase/firebase.json index 469e96e11..b81c02219 100644 --- a/firebase/firebase.json +++ b/firebase/firebase.json @@ -22,12 +22,12 @@ "function": "osmAuth-token" }, { - "source": "/redirect_web", - "function": "osmAuth-redirect_web" + "source": "/redirectweb", + "function": "osmAuth-redirectweb" }, { - "source": "/token_web", - "function": "osmAuth-token_web" + "source": "/tokenweb", + "function": "osmAuth-tokenweb" } ] }, diff --git a/firebase/functions/src/index.ts b/firebase/functions/src/index.ts index 6bb56e98c..c448cadc2 100644 --- a/firebase/functions/src/index.ts +++ b/firebase/functions/src/index.ts @@ -8,7 +8,7 @@ admin.initializeApp(); // all functions are bundled together. It's less than ideal, but it does not // seem possible to split them using the split system for multiple sites from // https://firebase.google.com/docs/hosting/multisites -import {redirect, token, redirect_web, token_web} from './osm_auth'; +import {redirect, token, redirectweb, tokenweb} from './osm_auth'; import { formatProjectTopic, formatUserName } from './utils'; exports.osmAuth = {}; @@ -23,12 +23,12 @@ exports.osmAuth.token = functions.https.onRequest((req, res) => { token(req, res, admin); }); -exports.osmAuth.redirect_web = functions.https.onRequest((req, res) => { - redirect_web(req, res); +exports.osmAuth.redirectweb = functions.https.onRequest((req, res) => { + redirectweb(req, res); }); -exports.osmAuth.token_web = functions.https.onRequest((req, res) => { - token_web(req, res, admin); +exports.osmAuth.tokenweb = functions.https.onRequest((req, res) => { + tokenweb(req, res, admin); }); /* diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 80c157774..9cc853a2d 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -101,7 +101,7 @@ export const redirect = (req: any, res: any) => { }); }; -export const redirect_web = (req: any, res: any) => { +export const redirectweb = (req: any, res: any) => { const oauth2 = osmOAuth2ClientWeb(); cookieParser()(req, res, () => { @@ -233,7 +233,7 @@ export const token = async (req: any, res: any, admin: any) => { }; -export const token_web = async (req: any, res: any, admin: any) => { +export const tokenweb = async (req: any, res: any, admin: any) => { const oauth2 = osmOAuth2ClientWeb(); try { @@ -244,7 +244,7 @@ export const token_web = async (req: any, res: any, admin: any) => { ); functions.logger.log('Received state:', req.query.state); // FIXME: For security, we need to check the cookie that was set - // in the /redirect_web function on the user's browser. + // in the /redirectweb function on the user's browser. // However, there seems to be a bug in firebase around this. // https://github.com/firebase/firebase-functions/issues/544 // and linked SO question From e2c22ba9b670e8442a0ebab7ea4487d7c897a153 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 15 May 2025 10:55:37 +0200 Subject: [PATCH 06/35] fix: add doc --- firebase/functions/src/osm_auth.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 9cc853a2d..b55613439 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -53,6 +53,11 @@ function osmOAuth2Client() { return simpleOAuth2.create(credentials); } +/** + * Creates a configured simple-oauth2 client for OSM for the web app. + * Configure the `osm.client_id_web` and `osm.client_secret_web` + * Google Cloud environment variables for the values below to exist + */ function osmOAuth2ClientWeb() { const credentials = { client: { From 261533105ee7a083a478b4ff0c3f7f73491dc4e5 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 19 May 2025 13:03:44 +0200 Subject: [PATCH 07/35] refactor: redirect and token --- firebase/functions/src/osm_auth.ts | 166 ++++++----------------------- 1 file changed, 34 insertions(+), 132 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index b55613439..7e721033d 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -38,7 +38,7 @@ const OSM_API_URL = functions.config().osm?.api_url; * Configure the `osm.client_id` and `osm.client_secret` * Google Cloud environment variables for the values below to exist */ -function osmOAuth2Client() { +function osmOAuth2Client(client_id, client_secret) { const credentials = { client: { id: functions.config().osm?.client_id, @@ -53,26 +53,6 @@ function osmOAuth2Client() { return simpleOAuth2.create(credentials); } -/** - * Creates a configured simple-oauth2 client for OSM for the web app. - * Configure the `osm.client_id_web` and `osm.client_secret_web` - * Google Cloud environment variables for the values below to exist - */ -function osmOAuth2ClientWeb() { - const credentials = { - client: { - id: functions.config().osm?.client_id_web, - secret: functions.config().osm?.client_secret_web, - }, - auth: { - tokenHost: OSM_API_URL, - tokenPath: '/oauth2/token', - authorizePath: '/oauth2/authorize', - }, - }; - return simpleOAuth2.create(credentials); -} - /** * Redirects the User to the OSM authentication consent screen. * Also the '__session' cookie is set for later state verification. @@ -80,8 +60,8 @@ function osmOAuth2ClientWeb() { * NOT a webview inside MapSwipe, as this would break the promise of * OAuth that we do not touch their OSM credentials */ -export const redirect = (req: any, res: any) => { - const oauth2 = osmOAuth2Client(); +function redirect2OsmOauth(redirect_uri, client_id, client_secret) { + const oauth2 = osmOAuth2Client(client_id, client_secret); cookieParser()(req, res, () => { const state = @@ -97,43 +77,31 @@ export const redirect = (req: any, res: any) => { httpOnly: true, }); const redirectUri = oauth2.authorizationCode.authorizeURL({ - redirect_uri: OAUTH_REDIRECT_URI, + redirect_uri: redirect_uri, scope: OAUTH_SCOPES, state: state, }); functions.logger.log('Redirecting to:', redirectUri); res.redirect(redirectUri); }); +} + +export const redirect = (req: any, res: any) => { + const redirect_uri = OAUTH_REDIRECT_URI; + const client_id = functions.config().osm?.client_id; + const client_secret = functions.config().osm?.client_secret; + redirect2OsmOauth(redirect_uri, client_id, client_secret); }; export const redirectweb = (req: any, res: any) => { - const oauth2 = osmOAuth2ClientWeb(); - - cookieParser()(req, res, () => { - const state = - req.cookies.state || crypto.randomBytes(20).toString('hex'); - functions.logger.log('Setting verification state:', state); - // the cookie MUST be called __session for hosted functions not to - // strip it from incoming requests - // (https://firebase.google.com/docs/hosting/manage-cache#using_cookies) - res.cookie('__session', state.toString(), { - // cookie is valid for 1 hour - maxAge: 3600000, - secure: true, - httpOnly: true, - }); - const redirectUri = oauth2.authorizationCode.authorizeURL({ - redirect_uri: OAUTH_REDIRECT_URI_WEB, - scope: OAUTH_SCOPES, - state: state, - }); - functions.logger.log('Redirecting to:', redirectUri); - res.redirect(redirectUri); - }); + const redirect_uri = OAUTH_REDIRECT_URI_WEB; + const client_id = functions.config().osm?.client_id_web; + const client_secret = functions.config().osm?.client_secret_web; + redirect2OsmOauth(redirect_uri, client_id, client_secret); }; /** - * The OSM OAuth endpoing does not give us any info about the user, + * The OSM OAuth endpoint does not give us any info about the user, * so we need to get the user profile from this endpoint */ async function getOSMProfile(accessToken: string) { @@ -155,8 +123,8 @@ async function getOSMProfile(accessToken: string) { * The Firebase custom auth token, display name, photo URL and OSM access * token are sent back to the app via a deeplink redirect. */ -export const token = async (req: any, res: any, admin: any) => { - const oauth2 = osmOAuth2Client(); +function fbToken(redirect_uri, osm_login_link, client_id, client_web) { + const oauth2 = osmOAuth2Client(client_id, client_web); try { return cookieParser()(req, res, async () => { @@ -187,7 +155,7 @@ export const token = async (req: any, res: any, admin: any) => { // this doesn't work results = await oauth2.authorizationCode.getToken({ code: req.query.code, - redirect_uri: OAUTH_REDIRECT_URI, + redirect_uri: redirect_uri, scope: OAUTH_SCOPES, state: req.query.state, }); @@ -225,7 +193,7 @@ export const token = async (req: any, res: any, admin: any) => { ); // build a deep link so we can send the token back to the app // from the browser - const signinUrl = `${APP_OSM_LOGIN_DEEPLINK}?token=${firebaseToken}`; + const signinUrl = `${osm_login_link}?token=${firebaseToken}`; functions.logger.log('redirecting user to', signinUrl); res.redirect(signinUrl); }); @@ -235,89 +203,23 @@ export const token = async (req: any, res: any, admin: any) => { // back into the app to allow the user to take action return res.json({ error: error.toString() }); } -}; - -export const tokenweb = async (req: any, res: any, admin: any) => { - const oauth2 = osmOAuth2ClientWeb(); - - try { - return cookieParser()(req, res, async () => { - functions.logger.log( - 'Received verification state:', - req.cookies.__session, - ); - functions.logger.log('Received state:', req.query.state); - // FIXME: For security, we need to check the cookie that was set - // in the /redirectweb function on the user's browser. - // However, there seems to be a bug in firebase around this. - // https://github.com/firebase/firebase-functions/issues/544 - // and linked SO question - // firebase docs mention the need for a cookie middleware, but there - // is no info about it :( - // cross site cookies don't seem to be the issue - // WE just need to make sure the domain set on the cookies is right - if (!req.cookies.__session) { - throw new Error('State cookie not set or expired. Maybe you took too long to authorize. Please try again.'); - } else if (req.cookies.__session !== req.query.state) { - throw new Error('State validation failed'); - } - functions.logger.log('Received auth code:', req.query.code); - let results; - - try { - // TODO: try adding auth data to request headers if - // this doesn't work - results = await oauth2.authorizationCode.getToken({ - code: req.query.code, - redirect_uri: OAUTH_REDIRECT_URI_WEB, - scope: OAUTH_SCOPES, - state: req.query.state, - }); - } catch (error: any) { - functions.logger.log('Auth token error', error, error.data.res.req); - } - // why is token called twice? - functions.logger.log( - 'Auth code exchange result received:', - results, - ); +} - // We have an OSM access token and the user identity now. - const accessToken = results && results.access_token; - if (accessToken === undefined) { - throw new Error( - 'Could not get an access token from OpenStreetMap', - ); - } - // get the OSM user id and display_name - const { id, display_name } = await getOSMProfile(accessToken); - functions.logger.log('osmuser:', id, display_name); - if (id === undefined) { - // this should not happen, but help guard against creating - // invalid accounts - throw new Error('Could not obtain an account id from OSM'); - } +export const token = async (req: any, res: any, admin: any) => { + const redirect_uri = OAUTH_REDIRECT_URI; + const osm_login_link = APP_OSM_LOGIN_DEEPLINK; + const client_id = functions.config().osm?.client_id; + const client_secret = functions.config().osm?.client_secret; + fbToken(redirect_uri, osm_login_link, client_id, client_secret); +}; - // Create a Firebase account and get the Custom Auth Token. - const firebaseToken = await createFirebaseAccount( - admin, - id, - display_name, - accessToken, - ); - // build a deep link so we can send the token back to the app - // from the browser - const signinUrl = `${APP_OSM_LOGIN_DEEPLINK_WEB}?token=${firebaseToken}`; - functions.logger.log('redirecting user to', signinUrl); - res.redirect(signinUrl); - }); - } catch (error: any) { - // FIXME: this should show up in the user's browser as a bit of text - // We should figure out the various error codes available and feed them - // back into the app to allow the user to take action - return res.json({ error: error.toString() }); - } +export const tokenweb = async (req: any, res: any, admin: any) => { + const redirect_uri = OAUTH_REDIRECT_URI_WEB; + const osm_login_link = APP_OSM_LOGIN_DEEPLINK_WEB; + const client_id = functions.config().osm?.client_id_web; + const client_secret = functions.config().osm?.client_secret_web; + fbToken(redirect_uri, osm_login_link, client_id, client_secret); }; /** From 515190c54694e1d5e0373a35917d94c37f534ebf Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 19 May 2025 13:11:48 +0200 Subject: [PATCH 08/35] fix: pass req res admin --- firebase/functions/src/osm_auth.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 7e721033d..584e4df65 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -60,7 +60,7 @@ function osmOAuth2Client(client_id, client_secret) { * NOT a webview inside MapSwipe, as this would break the promise of * OAuth that we do not touch their OSM credentials */ -function redirect2OsmOauth(redirect_uri, client_id, client_secret) { +function redirect2OsmOauth(req, res, redirect_uri, client_id, client_secret) { const oauth2 = osmOAuth2Client(client_id, client_secret); cookieParser()(req, res, () => { @@ -90,14 +90,14 @@ export const redirect = (req: any, res: any) => { const redirect_uri = OAUTH_REDIRECT_URI; const client_id = functions.config().osm?.client_id; const client_secret = functions.config().osm?.client_secret; - redirect2OsmOauth(redirect_uri, client_id, client_secret); + redirect2OsmOauth(req, res, redirect_uri, client_id, client_secret); }; export const redirectweb = (req: any, res: any) => { const redirect_uri = OAUTH_REDIRECT_URI_WEB; const client_id = functions.config().osm?.client_id_web; const client_secret = functions.config().osm?.client_secret_web; - redirect2OsmOauth(redirect_uri, client_id, client_secret); + redirect2OsmOauth(req, res, redirect_uri, client_id, client_secret); }; /** @@ -123,7 +123,7 @@ async function getOSMProfile(accessToken: string) { * The Firebase custom auth token, display name, photo URL and OSM access * token are sent back to the app via a deeplink redirect. */ -function fbToken(redirect_uri, osm_login_link, client_id, client_web) { +function fbToken(req, res, admin, redirect_uri, osm_login_link, client_id, client_web) { const oauth2 = osmOAuth2Client(client_id, client_web); try { @@ -211,7 +211,7 @@ export const token = async (req: any, res: any, admin: any) => { const osm_login_link = APP_OSM_LOGIN_DEEPLINK; const client_id = functions.config().osm?.client_id; const client_secret = functions.config().osm?.client_secret; - fbToken(redirect_uri, osm_login_link, client_id, client_secret); + fbToken(req, res, admin, redirect_uri, osm_login_link, client_id, client_secret); }; export const tokenweb = async (req: any, res: any, admin: any) => { @@ -219,7 +219,7 @@ export const tokenweb = async (req: any, res: any, admin: any) => { const osm_login_link = APP_OSM_LOGIN_DEEPLINK_WEB; const client_id = functions.config().osm?.client_id_web; const client_secret = functions.config().osm?.client_secret_web; - fbToken(redirect_uri, osm_login_link, client_id, client_secret); + fbToken(req, res, admin, redirect_uri, osm_login_link, client_id, client_secret); }; /** From 52a00542a3e58ef8b649ae4d167875c8cc38f51a Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 19 May 2025 13:15:49 +0200 Subject: [PATCH 09/35] style: remove blank line padding --- firebase/functions/src/osm_auth.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 584e4df65..6fadc565f 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -203,7 +203,6 @@ function fbToken(req, res, admin, redirect_uri, osm_login_link, client_id, clien // back into the app to allow the user to take action return res.json({ error: error.toString() }); } - } export const token = async (req: any, res: any, admin: any) => { From e04f69459acbc95d35d1a5e3b071ee22fe6771b8 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 19 May 2025 13:38:24 +0200 Subject: [PATCH 10/35] fix: typing --- firebase/functions/src/osm_auth.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 6fadc565f..c344ced86 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -38,11 +38,11 @@ const OSM_API_URL = functions.config().osm?.api_url; * Configure the `osm.client_id` and `osm.client_secret` * Google Cloud environment variables for the values below to exist */ -function osmOAuth2Client(client_id, client_secret) { +function osmOAuth2Client(client_id: any, client_secret: any) { const credentials = { client: { - id: functions.config().osm?.client_id, - secret: functions.config().osm?.client_secret, + id: client_id, + secret: client_secret, }, auth: { tokenHost: OSM_API_URL, @@ -60,7 +60,7 @@ function osmOAuth2Client(client_id, client_secret) { * NOT a webview inside MapSwipe, as this would break the promise of * OAuth that we do not touch their OSM credentials */ -function redirect2OsmOauth(req, res, redirect_uri, client_id, client_secret) { +function redirect2OsmOauth(req: any, res: any, redirect_uri: string, client_id: string, client_secret: string) { const oauth2 = osmOAuth2Client(client_id, client_secret); cookieParser()(req, res, () => { @@ -123,7 +123,7 @@ async function getOSMProfile(accessToken: string) { * The Firebase custom auth token, display name, photo URL and OSM access * token are sent back to the app via a deeplink redirect. */ -function fbToken(req, res, admin, redirect_uri, osm_login_link, client_id, client_web) { +function fbToken(req: any , res: any, admin: any, redirect_uri: string, osm_login_link: string, client_id: string, client_web: string) { const oauth2 = osmOAuth2Client(client_id, client_web); try { From 3944e677682585d1f0fe35b507f7fea85bbaad90 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 19 May 2025 13:41:24 +0200 Subject: [PATCH 11/35] fix: remove space before , --- firebase/functions/src/osm_auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index c344ced86..27e4f562a 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -123,7 +123,7 @@ async function getOSMProfile(accessToken: string) { * The Firebase custom auth token, display name, photo URL and OSM access * token are sent back to the app via a deeplink redirect. */ -function fbToken(req: any , res: any, admin: any, redirect_uri: string, osm_login_link: string, client_id: string, client_web: string) { +function fbToken(req: any, res: any, admin: any, redirect_uri: string, osm_login_link: string, client_id: string, client_web: string) { const oauth2 = osmOAuth2Client(client_id, client_web); try { From 95d0c56bc6f950ebeabb7ba12c9ccb43e61bb65e Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 18:47:17 +0200 Subject: [PATCH 12/35] fix: avoid overwriting existing osm profiles on sign in --- firebase/functions/src/osm_auth.ts | 46 ++++++++++++++++++++++-------- mapswipe_workers/requirements.txt | 1 + 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 27e4f562a..18d165d6a 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -236,23 +236,19 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // with a variable length. const uid = `osm:${osmID}`; + // check if profile exists on Firebase Realtime Database + const profileExists = await admin + .database() + .ref(`v2/users/osm:${id}/`) + .get() + .then((snapshot: any) => { return snapshot.exists()}); + // Save the access token to the Firebase Realtime Database. const databaseTask = admin .database() .ref(`v2/OSMAccessToken/${uid}`) .set(accessToken); - const profileTask = admin - .database() - .ref(`v2/users/${uid}/`) - .set({ - created: new Date().toISOString(), - groupContributionCount: 0, - projectContributionCount: 0, - taskContributionCount: 0, - displayName, - }); - // Create or update the firebase user account. // This does not login the user on the app, it just ensures that a firebase // user account (linked to the OSM account) exists. @@ -272,8 +268,34 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a throw error; }); + // Only update display name if profile exists, else create profile + const profileUpdateTask = admin + .database() + .ref(`v2/users/${uid}/`) + .update({ displayName }) + + const profileCreationTask = admin + .database() + .ref(`v2/users/${uid}/`) + .set({ + created: new Date().toISOString(), + groupContributionCount: 0, + projectContributionCount: 0, + taskContributionCount: 0, + displayName, + }); + } + + const tasks = [userCreationTask, databaseTask] + + if (profileExists) { + tasks.push(profileUpdateTask) + } else { + tasks.push(profileCreationTask) + } + // Wait for all async task to complete then generate and return a custom auth token. - await Promise.all([userCreationTask, databaseTask, profileTask]); + await Promise.all(tasks); // Create a Firebase custom auth token. functions.logger.log('In createFirebaseAccount: createCustomToken'); let authToken; diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 588754060..13462eaa5 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -5,6 +5,7 @@ firebase-admin==6.0.0 flake8==3.8.3 geojson==3.0.1 mapswipe-workers==3.0 +numpy==1.26.4 pandas==1.5.2 pre-commit==2.9.2 psycopg2-binary==2.9.3 From 29ce576e95a874c3f8caed9732f42759dcd94bb4 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 18:52:21 +0200 Subject: [PATCH 13/35] revert change to requirements --- mapswipe_workers/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 13462eaa5..588754060 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -5,7 +5,6 @@ firebase-admin==6.0.0 flake8==3.8.3 geojson==3.0.1 mapswipe-workers==3.0 -numpy==1.26.4 pandas==1.5.2 pre-commit==2.9.2 psycopg2-binary==2.9.3 From 57385ef430019cfdcc5ec1e56bdb5f7f8da89b76 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 19:03:18 +0200 Subject: [PATCH 14/35] fix: clean parentheses --- firebase/functions/src/osm_auth.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 18d165d6a..0c8f0dd76 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -239,7 +239,7 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // check if profile exists on Firebase Realtime Database const profileExists = await admin .database() - .ref(`v2/users/osm:${id}/`) + .ref(`v2/users/${uid}/`) .get() .then((snapshot: any) => { return snapshot.exists()}); @@ -272,7 +272,7 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a const profileUpdateTask = admin .database() .ref(`v2/users/${uid}/`) - .update({ displayName }) + .update({ displayName }); const profileCreationTask = admin .database() @@ -284,14 +284,13 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a taskContributionCount: 0, displayName, }); - } - const tasks = [userCreationTask, databaseTask] + const tasks = [userCreationTask, databaseTask]; if (profileExists) { - tasks.push(profileUpdateTask) + tasks.push(profileUpdateTask); } else { - tasks.push(profileCreationTask) + tasks.push(profileCreationTask); } // Wait for all async task to complete then generate and return a custom auth token. From a8d0ae6152e62b82060a71ce4bbb372d9f116ceb Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 19:09:10 +0200 Subject: [PATCH 15/35] fix: formatting --- firebase/functions/src/osm_auth.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 0c8f0dd76..76d820b2e 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -241,7 +241,9 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a .database() .ref(`v2/users/${uid}/`) .get() - .then((snapshot: any) => { return snapshot.exists()}); + .then((snapshot: any) => { + return snapshot.exists(); + }); // Save the access token to the Firebase Realtime Database. const databaseTask = admin @@ -270,9 +272,9 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // Only update display name if profile exists, else create profile const profileUpdateTask = admin - .database() - .ref(`v2/users/${uid}/`) - .update({ displayName }); + .database() + .ref(`v2/users/${uid}/`) + .update({ displayName }); const profileCreationTask = admin .database() From a8c662b43a8e7b931a89cb16cd5a737086f8ce9d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 19:32:48 +0200 Subject: [PATCH 16/35] fix: use once instead of get --- firebase/functions/src/osm_auth.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 76d820b2e..c19d0bc0d 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -237,13 +237,12 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a const uid = `osm:${osmID}`; // check if profile exists on Firebase Realtime Database - const profileExists = await admin + const snapshot = await admin .database() .ref(`v2/users/${uid}/`) - .get() - .then((snapshot: any) => { - return snapshot.exists(); - }); + .once() + + const profileExists = snapshot.exists() // Save the access token to the Firebase Realtime Database. const databaseTask = admin From 0aace7f39b2ba9fc04df709193604ffd8a837765 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 19:38:01 +0200 Subject: [PATCH 17/35] reuse profileRef and add semicolons --- firebase/functions/src/osm_auth.ts | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index c19d0bc0d..b600d0c3a 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -236,13 +236,11 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // with a variable length. const uid = `osm:${osmID}`; - // check if profile exists on Firebase Realtime Database - const snapshot = await admin - .database() - .ref(`v2/users/${uid}/`) - .once() + const profileRef = admin.database().ref(`v2/users/${uid}/`); - const profileExists = snapshot.exists() + // check if profile exists on Firebase Realtime Database + const snapshot = await profileRef.once(); + const profileExists = snapshot.exists(); // Save the access token to the Firebase Realtime Database. const databaseTask = admin @@ -270,14 +268,8 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a }); // Only update display name if profile exists, else create profile - const profileUpdateTask = admin - .database() - .ref(`v2/users/${uid}/`) - .update({ displayName }); - - const profileCreationTask = admin - .database() - .ref(`v2/users/${uid}/`) + const profileUpdateTask = profileRef.update({ displayName }); + const profileCreationTask = profileRef .set({ created: new Date().toISOString(), groupContributionCount: 0, From bbfb5ad2ce84ab2a189acdaef6da26aad12bbfd2 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 20:01:06 +0200 Subject: [PATCH 18/35] fix: add arg to once --- firebase/functions/src/osm_auth.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index b600d0c3a..3328f7079 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -236,10 +236,10 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // with a variable length. const uid = `osm:${osmID}`; - const profileRef = admin.database().ref(`v2/users/${uid}/`); + const profileRef = admin.database().ref(`v2/users/${uid}`); // check if profile exists on Firebase Realtime Database - const snapshot = await profileRef.once(); + const snapshot = await profileRef.once('value'); const profileExists = snapshot.exists(); // Save the access token to the Firebase Realtime Database. @@ -268,7 +268,7 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a }); // Only update display name if profile exists, else create profile - const profileUpdateTask = profileRef.update({ displayName }); + const profileUpdateTask = profileRef.update({ displayName: displayName }); const profileCreationTask = profileRef .set({ created: new Date().toISOString(), From 09a955df77c78a859b4f8f5be2b5a3e344b218c2 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 20 May 2025 20:13:21 +0200 Subject: [PATCH 19/35] add logging --- firebase/functions/src/osm_auth.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 3328f7079..620b247f2 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -240,6 +240,7 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // check if profile exists on Firebase Realtime Database const snapshot = await profileRef.once('value'); + functions.logger.log("Snapshot value:", snapshot.val()); const profileExists = snapshot.exists(); // Save the access token to the Firebase Realtime Database. @@ -281,8 +282,10 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a const tasks = [userCreationTask, databaseTask]; if (profileExists) { + functions.logger.log('Sign in to existing OSM profile'); tasks.push(profileUpdateTask); } else { + functions.logger.log('Sign up new OSM profile'); tasks.push(profileCreationTask); } From 61bb66ec9dfe92363c1e24636b8c3eb4c1973d41 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 21 May 2025 08:51:44 +0200 Subject: [PATCH 20/35] fix: use singlequote --- firebase/functions/src/osm_auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 620b247f2..bbc54c34e 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -240,7 +240,7 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // check if profile exists on Firebase Realtime Database const snapshot = await profileRef.once('value'); - functions.logger.log("Snapshot value:", snapshot.val()); + functions.logger.log('Snapshot value:', snapshot.val()); const profileExists = snapshot.exists(); // Save the access token to the Firebase Realtime Database. From 85710d57e32cf375ea320eb4911f6609ae1d104d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 21 May 2025 09:01:22 +0200 Subject: [PATCH 21/35] move profile tasks definition --- firebase/functions/src/osm_auth.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index bbc54c34e..95d9169fa 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -269,23 +269,22 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a }); // Only update display name if profile exists, else create profile - const profileUpdateTask = profileRef.update({ displayName: displayName }); - const profileCreationTask = profileRef - .set({ - created: new Date().toISOString(), - groupContributionCount: 0, - projectContributionCount: 0, - taskContributionCount: 0, - displayName, - }); - const tasks = [userCreationTask, databaseTask]; if (profileExists) { functions.logger.log('Sign in to existing OSM profile'); + const profileUpdateTask = profileRef.update({ displayName: displayName }); tasks.push(profileUpdateTask); } else { functions.logger.log('Sign up new OSM profile'); + const profileCreationTask = profileRef + .set({ + created: new Date().toISOString(), + groupContributionCount: 0, + projectContributionCount: 0, + taskContributionCount: 0, + displayName, + }); tasks.push(profileCreationTask); } From 3430784817d3a94d0228928322fd617629eefe10 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 21 May 2025 09:05:14 +0200 Subject: [PATCH 22/35] fix indentation --- firebase/functions/src/osm_auth.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 95d9169fa..9f0b286f2 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -278,13 +278,13 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a } else { functions.logger.log('Sign up new OSM profile'); const profileCreationTask = profileRef - .set({ - created: new Date().toISOString(), - groupContributionCount: 0, - projectContributionCount: 0, - taskContributionCount: 0, - displayName, - }); + .set({ + created: new Date().toISOString(), + groupContributionCount: 0, + projectContributionCount: 0, + taskContributionCount: 0, + displayName, + }); tasks.push(profileCreationTask); } From ed806f6503d6008329f79fd940e09deb468d7e05 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 21 May 2025 09:33:09 +0200 Subject: [PATCH 23/35] fix: clean up --- firebase/functions/src/osm_auth.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/firebase/functions/src/osm_auth.ts b/firebase/functions/src/osm_auth.ts index 9f0b286f2..9953f2ea9 100644 --- a/firebase/functions/src/osm_auth.ts +++ b/firebase/functions/src/osm_auth.ts @@ -240,7 +240,6 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a // check if profile exists on Firebase Realtime Database const snapshot = await profileRef.once('value'); - functions.logger.log('Snapshot value:', snapshot.val()); const profileExists = snapshot.exists(); // Save the access token to the Firebase Realtime Database. @@ -268,9 +267,8 @@ async function createFirebaseAccount(admin: any, osmID: any, displayName: any, a throw error; }); - // Only update display name if profile exists, else create profile + // If profile exists, only update displayName -- else create new user profile const tasks = [userCreationTask, databaseTask]; - if (profileExists) { functions.logger.log('Sign in to existing OSM profile'); const profileUpdateTask = profileRef.update({ displayName: displayName }); From a758dc1c42707639dc394c99ee8bec1e66a29d20 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 21 May 2025 13:28:00 +0200 Subject: [PATCH 24/35] docs(osm-login-web): add web osm login to docs --- firebase/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/firebase/README.md b/firebase/README.md index fed47268d..c3381abc9 100644 --- a/firebase/README.md +++ b/firebase/README.md @@ -20,6 +20,11 @@ expose the authentication functions publicly. * `firebase deploy --only functions,hosting` * `firebase deploy --only database:rules` +## Deploy with Makefile +You can also deploy the changes to Firebase using make: +* Make sure to remove the firebase_deploy docker image first: `docker rmi python-mapswipe-workers-firebase_deploy` +* `make update_firebase_functions_and_db_rules` + ## Notes on OAuth (OSM login) Refer to [the notes in the app repository](https://github.com/mapswipe/mapswipe/blob/master/docs/osm_login.md). @@ -30,12 +35,16 @@ Some specifics about the related functions: - Before deploying, set the required firebase config values in environment: FIXME: replace env vars with config value names - OSM_OAUTH_REDIRECT_URI `osm.redirect_uri`: `https://dev-auth.mapswipe.org/token` or `https://auth.mapswipe.org/token` + - OSM_OAUTH_REDIRECT_URI_WEB: `https://dev-auth.mapswipe.org/tokenweb` or `https://auth.mapswipe.org/tokenweb` - OSM_OAUTH_APP_LOGIN_LINK `osm.app_login_link`: 'devmapswipe://login/osm' or 'mapswipe://login/osm' + - OSM_OAUTH_APP_LOGIN_LINK_WEB: `https://web.mapswipe.org/dev/#/osm-callback` or `https://web.mapswipe.org/#/osm-callback` - OSM_OAUTH_API_URL `osm.api_url`: 'https://master.apis.dev.openstreetmap.org/' or 'https://www.openstreetmap.org/' (include the trailing slash) - OSM_OAUTH_CLIENT_ID `osm.client_id`: find it on the OSM application page - OSM_OAUTH_CLIENT_SECRET `osm.client_secret`: same as above. Note that this can only be seen once when the application is created. Do not lose it! + - OSM_OAUTH_CLIENT_ID_WEB: This is the ID of a __different__ registered OSM OAuth client for the web version that needs to have `https://dev-auth.mapswipe.org/tokenweb` or `https://auth.mapswipe.org/tokenweb` set as redirect URI. + - OSM_OAUTH_CLIENT_SECRET_WEB: This is the secret of the OSM OAuth client for MapSwipe web version. - Deploy the functions as explained above - Expose the functions publicly through firebase hosting, this is done in `/firebase/firebase.json` under the `hosting` key. From d256198504db1b0609ecc83a67a9c66da1c105b8 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 12 Jun 2025 10:47:54 +0545 Subject: [PATCH 25/35] fix(sql): fix typo in comment in initdb.sql --- mapswipe_workers/tests/integration/set_up_db.sql | 2 +- postgres/initdb.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index f954d3a8c..6a8d1edd5 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS groups ( required_count int, progress int, project_type_specifics json, - -- total_area & time_spent_max_allowed are maintaned and used by aggregated module + -- total_area & time_spent_max_allowed are maintained and used by aggregated module total_area float DEFAULT NULL, time_spent_max_allowed float DEFAULT NULL, PRIMARY KEY (project_id, group_id), diff --git a/postgres/initdb.sql b/postgres/initdb.sql index f954d3a8c..6a8d1edd5 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS groups ( required_count int, progress int, project_type_specifics json, - -- total_area & time_spent_max_allowed are maintaned and used by aggregated module + -- total_area & time_spent_max_allowed are maintained and used by aggregated module total_area float DEFAULT NULL, time_spent_max_allowed float DEFAULT NULL, PRIMARY KEY (project_id, group_id), From d4805a0dc84dab9ce199b1b007b7860e8c30950c Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 12 Jun 2025 10:49:24 +0545 Subject: [PATCH 26/35] feat(django): add support for unaccent search for usergroups --- django/apps/existing_database/filters.py | 2 +- mapswipe_workers/tests/integration/set_up_db.sql | 1 + postgres/initdb.sql | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django/apps/existing_database/filters.py b/django/apps/existing_database/filters.py index 096f2f5a1..98227d94b 100644 --- a/django/apps/existing_database/filters.py +++ b/django/apps/existing_database/filters.py @@ -38,7 +38,7 @@ class UserGroupFilter: def filter_search(self, queryset): if self.search: queryset = queryset.filter( - name__icontains=self.search, + name__unaccent__icontains=self.search, ) return queryset diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index 6a8d1edd5..b2b23f328 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -1,5 +1,6 @@ -- noinspection SqlNoDataSourceInspectionForFile CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS unaccent; CREATE TABLE IF NOT EXISTS projects ( created timestamp, diff --git a/postgres/initdb.sql b/postgres/initdb.sql index 6a8d1edd5..b2b23f328 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -1,5 +1,6 @@ -- noinspection SqlNoDataSourceInspectionForFile CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS unaccent; CREATE TABLE IF NOT EXISTS projects ( created timestamp, From 4242d7bea4a12882420aaf8f8932e0752d4e5f4f Mon Sep 17 00:00:00 2001 From: tnagorra Date: Thu, 26 Jun 2025 09:48:08 +0545 Subject: [PATCH 27/35] feat(validate_image): add validate image project & tutorial creation --- .../mapswipe_workers/definitions.py | 5 + .../mapswipe_workers/firebase/firebase.py | 2 + .../project_types/__init__.py | 4 + .../project_types/validate_image/__init__.py | 0 .../project_types/validate_image/project.py | 108 ++++++++++++++++++ .../project_types/validate_image/tutorial.py | 80 +++++++++++++ 6 files changed, 199 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/project_types/validate_image/__init__.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/validate_image/project.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/validate_image/tutorial.py diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index aa32d3aac..afa7cb6ff 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -137,6 +137,7 @@ class ProjectType(Enum): MEDIA_CLASSIFICATION = 5 DIGITIZATION = 6 STREET = 7 + VALIDATE_IMAGE = 10 @property def constructor(self): @@ -149,6 +150,7 @@ def constructor(self): FootprintProject, MediaClassificationProject, StreetProject, + ValidateImageProject, ) project_type_classes = { @@ -159,6 +161,7 @@ def constructor(self): 5: MediaClassificationProject, 6: DigitizationProject, 7: StreetProject, + 10: ValidateImageProject, } return project_type_classes[self.value] @@ -171,6 +174,7 @@ def tutorial(self): CompletenessTutorial, FootprintTutorial, StreetTutorial, + ValidateImageTutorial, ) project_type_classes = { @@ -179,5 +183,6 @@ def tutorial(self): 3: ChangeDetectionTutorial, 4: CompletenessTutorial, 7: StreetTutorial, + 10: ValidateImageTutorial, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/firebase/firebase.py b/mapswipe_workers/mapswipe_workers/firebase/firebase.py index 809b6c801..b91256985 100644 --- a/mapswipe_workers/mapswipe_workers/firebase/firebase.py +++ b/mapswipe_workers/mapswipe_workers/firebase/firebase.py @@ -14,6 +14,7 @@ def save_project_to_firebase(self, project): # if a geometry exists in projects we want to delete it. # This geometry is not used in clients. project.pop("geometry", None) + # FIXME: We might need to pop images # save project self.ref.update({f"v2/projects/{project['projectId']}": project}) logger.info( @@ -82,6 +83,7 @@ def save_tutorial_to_firebase( tutorialDict.pop("raw_tasks", None) tutorialDict.pop("examplesFile", None) tutorialDict.pop("tutorial_tasks", None) + tutorialDict.pop("images", None) if not tutorial.projectId or tutorial.projectId == "": raise CustomError( diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index 9560c76ef..3fb4e722b 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -10,6 +10,8 @@ from .tile_map_service.classification.tutorial import ClassificationTutorial from .tile_map_service.completeness.project import CompletenessProject from .tile_map_service.completeness.tutorial import CompletenessTutorial +from .validate_image.project import ValidateImageProject +from .validate_image.tutorial import ValidateImageTutorial __all__ = [ "ClassificationProject", @@ -21,6 +23,8 @@ "MediaClassificationProject", "FootprintProject", "FootprintTutorial", + "ValidateImageProject", + "ValidateImageTutorial", "DigitizationProject", "StreetProject", "StreetTutorial", diff --git a/mapswipe_workers/mapswipe_workers/project_types/validate_image/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/validate_image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mapswipe_workers/mapswipe_workers/project_types/validate_image/project.py b/mapswipe_workers/mapswipe_workers/project_types/validate_image/project.py new file mode 100644 index 000000000..e0aee5f4d --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/validate_image/project.py @@ -0,0 +1,108 @@ +import math +from dataclasses import dataclass +from typing import Dict, List + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.firebase_to_postgres.transfer_results import ( + results_to_file, + save_results_to_postgres, + truncate_temp_results, +) +from mapswipe_workers.generate_stats.project_stats import ( + get_statistics_for_integer_result_project, +) +from mapswipe_workers.project_types.project import BaseGroup, BaseProject + + +@dataclass +class ValidateImageGroup(BaseGroup): + pass + + +@dataclass +class ValidateImageTask: + # TODO(tnagorra): We need to check if fileName should be saved on project + # NOTE: We do not need to add projectId and groupId so we are not extending BaseTask + + # NOTE: taskId is the sourceIdentifier + taskId: str + + fileName: str + url: str + + # NOTE: This is not required but required by the base class + geometry: str + + +class ValidateImageProject(BaseProject): + def __init__(self, project_draft): + super().__init__(project_draft) + self.groups: Dict[str, ValidateImageGroup] = {} + self.tasks: Dict[str, List[ValidateImageTask]] = {} # dict keys are group ids + + # NOTE: This is a standard structure defined on manager dashboard. + # It's derived from other formats like COCO. + # The transfromation is done in manager dashboard. + self.images = project_draft["images"] + + def save_tasks_to_firebase(self, projectId: str, tasks: dict): + firebase = Firebase() + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False) + + @staticmethod + def results_to_postgres(results: dict, project_id: str, filter_mode: bool): + """How to move the result data from firebase to postgres.""" + results_file, user_group_results_file = results_to_file(results, project_id) + truncate_temp_results() + save_results_to_postgres(results_file, project_id, filter_mode) + return user_group_results_file + + @staticmethod + def get_per_project_statistics(project_id, project_info): + """How to aggregate the project results.""" + return get_statistics_for_integer_result_project( + project_id, project_info, generate_hot_tm_geometries=False + ) + + def validate_geometries(self): + pass + + def save_to_files(self, project): + """We do not have any geometry so we pass here""" + pass + + def create_groups(self): + self.numberOfGroups = math.ceil(len(self.images) / self.groupSize) + for group_index in range(self.numberOfGroups): + self.groups[f"g{group_index + 100}"] = ValidateImageGroup( + projectId=self.projectId, + groupId=f"g{group_index + 100}", + progress=0, + finishedCount=0, + requiredCount=0, + numberOfTasks=self.groupSize, + ) + logger.info(f"{self.projectId} - create_groups - created groups dictionary") + + def create_tasks(self): + if len(self.groups) == 0: + raise ValueError("Groups needs to be created before tasks can be created.") + for group_id, group in self.groups.items(): + self.tasks[group_id] = [] + for i in range(self.groupSize): + # FIXME: We should try not to mutate values + image_metadata = self.images.pop() + task = ValidateImageTask( + taskId=image_metadata["sourceIdentifier"], + fileName=image_metadata["fileName"], + url=image_metadata["url"], + geometry="", + ) + self.tasks[group_id].append(task) + + # list now empty? if usual group size is not reached + # the actual number of tasks for the group is updated + if not self.images: + group.numberOfTasks = i + 1 + break diff --git a/mapswipe_workers/mapswipe_workers/project_types/validate_image/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/validate_image/tutorial.py new file mode 100644 index 000000000..b42b0be61 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/validate_image/tutorial.py @@ -0,0 +1,80 @@ +from dataclasses import dataclass + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.project_types.tutorial import BaseTutorial +from mapswipe_workers.project_types.validate_image.project import ( + ValidateImageGroup, + ValidateImageTask, +) + + +@dataclass +class ValidateImageTutorialTask(ValidateImageTask): + # TODO(tnagorra): Check if we need projectId and groupId in tutorial task + projectId: str + groupId: str + referenceAnswer: int + screen: int + + +class ValidateImageTutorial(BaseTutorial): + + def __init__(self, tutorial_draft): + # this will create the basis attributes + super().__init__(tutorial_draft) + + self.groups = dict() + self.tasks = dict() + self.images = tutorial_draft["images"] + + def create_tutorial_groups(self): + """Create group for the tutorial based on provided examples in images.""" + + # NOTE: The groupId must be a numeric 101. It's hardcoded in save_tutorial_to_firebase + group = ValidateImageGroup( + groupId=101, + projectId=self.projectId, + numberOfTasks=len(self.images), + progress=0, + finishedCount=0, + requiredCount=0, + ) + self.groups[101] = group + + logger.info( + f"{self.projectId} - create_tutorial_groups - created groups dictionary" + ) + + def create_tutorial_tasks(self): + """Create the tasks dict based on provided examples in geojson file.""" + task_list = [] + for image_metadata in self.images: + image_metadata = ValidateImageTutorialTask( + projectId=self.projectId, + groupId=101, + taskId=image_metadata["sourceIdentifier"], + fileName=image_metadata["fileName"], + url=image_metadata["url"], + geometry="", + referenceAnswer=image_metadata["referenceAnswer"], + screen=image_metadata["screen"], + ) + task_list.append(image_metadata) + + if task_list: + self.tasks[101] = task_list + else: + logger.info(f"group in project {self.projectId} is not valid.") + + logger.info( + f"{self.projectId} - create_tutorial_tasks - created tasks dictionary" + ) + + def save_tutorial(self): + firebase = Firebase() + firebase.save_tutorial_to_firebase( + self, self.groups, self.tasks, useCompression=False + ) + logger.info(self.tutorialDraftId) + firebase.drop_tutorial_draft(self.tutorialDraftId) From c28bea1e833d4c8356033f47bed5a636dea33fbd Mon Sep 17 00:00:00 2001 From: tnagorra Date: Thu, 26 Jun 2025 09:55:50 +0545 Subject: [PATCH 28/35] fix(docker-compose): add mapillary_api_key in tc docker compose --- docker-compose.tc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.tc.yaml b/docker-compose.tc.yaml index 210c09570..62475284e 100644 --- a/docker-compose.tc.yaml +++ b/docker-compose.tc.yaml @@ -33,6 +33,7 @@ x-mapswipe-workers: &base_mapswipe_workers SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' OSMCHA_API_KEY: '${OSMCHA_API_KEY}' + MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}' depends_on: - postgres volumes: From 3fb2aa213f94e4838c053bd707aa9f7fbe445bc8 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Thu, 26 Jun 2025 09:57:11 +0545 Subject: [PATCH 29/35] feat(validate_image): update form for validate image project & tutorial creation - add io-ts for validation (with typescript types) - fix conditional issue with street project --- .../app/Base/configs/projectTypes.ts | 2 + manager-dashboard/app/Base/styles.css | 1 + .../Calendar/CalendarDate/index.tsx | 2 +- .../app/components/Calendar/index.tsx | 2 +- .../app/components/DateRangeInput/index.tsx | 2 +- .../app/components/InputSection/styles.css | 2 +- manager-dashboard/app/utils/common.tsx | 4 +- .../app/views/NewProject/ImageInput/index.tsx | 73 ++++++ .../views/NewProject/ImageInput/styles.css | 5 + .../app/views/NewProject/index.tsx | 177 +++++++++++++- .../app/views/NewProject/styles.css | 8 + .../app/views/NewProject/utils.ts | 224 +++++++++++++----- .../views/NewTutorial/ImageInput/index.tsx | 92 +++++++ .../views/NewTutorial/ImageInput/styles.css | 5 + .../FootprintGeoJsonPreview/index.tsx | 2 +- .../ValidateImagePreview/index.tsx | 81 +++++++ .../ValidateImagePreview/styles.css | 37 +++ .../NewTutorial/ScenarioPageInput/index.tsx | 42 +++- .../app/views/NewTutorial/index.tsx | 183 ++++++++++++-- .../app/views/NewTutorial/styles.css | 7 + .../app/views/NewTutorial/utils.ts | 156 +++++++++++- manager-dashboard/package.json | 3 + manager-dashboard/yarn.lock | 15 ++ 23 files changed, 1031 insertions(+), 94 deletions(-) create mode 100644 manager-dashboard/app/views/NewProject/ImageInput/index.tsx create mode 100644 manager-dashboard/app/views/NewProject/ImageInput/styles.css create mode 100644 manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx create mode 100644 manager-dashboard/app/views/NewTutorial/ImageInput/styles.css create mode 100644 manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/index.tsx create mode 100644 manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css diff --git a/manager-dashboard/app/Base/configs/projectTypes.ts b/manager-dashboard/app/Base/configs/projectTypes.ts index e2f7f74eb..e6344d507 100644 --- a/manager-dashboard/app/Base/configs/projectTypes.ts +++ b/manager-dashboard/app/Base/configs/projectTypes.ts @@ -5,6 +5,7 @@ import { PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_STREET, PROJECT_TYPE_COMPLETENESS, + PROJECT_TYPE_VALIDATE_IMAGE, } from '#utils/common'; const PROJECT_CONFIG_NAME = process.env.REACT_APP_PROJECT_CONFIG_NAME as string; @@ -15,6 +16,7 @@ const mapswipeProjectTypeOptions: { }[] = [ { value: PROJECT_TYPE_BUILD_AREA, label: 'Find' }, { value: PROJECT_TYPE_FOOTPRINT, label: 'Validate' }, + { value: PROJECT_TYPE_VALIDATE_IMAGE, label: 'Validate Image' }, { value: PROJECT_TYPE_CHANGE_DETECTION, label: 'Compare' }, { value: PROJECT_TYPE_STREET, label: 'Street' }, { value: PROJECT_TYPE_COMPLETENESS, label: 'Completeness' }, diff --git a/manager-dashboard/app/Base/styles.css b/manager-dashboard/app/Base/styles.css index c746dc570..87052b68a 100644 --- a/manager-dashboard/app/Base/styles.css +++ b/manager-dashboard/app/Base/styles.css @@ -105,6 +105,7 @@ p { --height-mobile-preview-builarea-content: 30rem; --height-mobile-preview-footprint-content: 22rem; --height-mobile-preview-change-detection-content: 14rem; + --height-mobile-preview-validate-image-content: 22rem; --radius-popup-border: 0.25rem; --radius-scrollbar-border: 0.25rem; diff --git a/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx index 2b1fed1a0..2520a21f1 100644 --- a/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx +++ b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { _cs } from '@togglecorp/fujs'; import RawButton, { Props as RawButtonProps } from '../../RawButton'; -import { ymdToDateString, typedMemo } from '../../../utils/common.tsx'; +import { ymdToDateString, typedMemo } from '../../../utils/common'; import styles from './styles.css'; diff --git a/manager-dashboard/app/components/Calendar/index.tsx b/manager-dashboard/app/components/Calendar/index.tsx index d72f9054d..21bd7de4f 100644 --- a/manager-dashboard/app/components/Calendar/index.tsx +++ b/manager-dashboard/app/components/Calendar/index.tsx @@ -15,7 +15,7 @@ import Button from '../Button'; import NumberInput from '../NumberInput'; import SelectInput from '../SelectInput'; import useInputState from '../../hooks/useInputState'; -import { typedMemo } from '../../utils/common.tsx'; +import { typedMemo } from '../../utils/common'; import CalendarDate, { Props as CalendarDateProps } from './CalendarDate'; diff --git a/manager-dashboard/app/components/DateRangeInput/index.tsx b/manager-dashboard/app/components/DateRangeInput/index.tsx index 6442fc835..0b25782ee 100644 --- a/manager-dashboard/app/components/DateRangeInput/index.tsx +++ b/manager-dashboard/app/components/DateRangeInput/index.tsx @@ -19,7 +19,7 @@ import Button from '../Button'; import Popup from '../Popup'; import Calendar, { Props as CalendarProps } from '../Calendar'; import CalendarDate, { Props as CalendarDateProps } from '../Calendar/CalendarDate'; -import { ymdToDateString, dateStringToDate } from '../../utils/common.tsx'; +import { ymdToDateString, dateStringToDate } from '../../utils/common'; import { predefinedDateRangeOptions, diff --git a/manager-dashboard/app/components/InputSection/styles.css b/manager-dashboard/app/components/InputSection/styles.css index 0c0012c77..f729ff036 100644 --- a/manager-dashboard/app/components/InputSection/styles.css +++ b/manager-dashboard/app/components/InputSection/styles.css @@ -24,7 +24,7 @@ display: flex; flex-direction: column; border-radius: var(--radius-card-border); - gap: var(--spacing-extra-large); + gap: var(--spacing-large); background-color: var(--color-foreground); padding: var(--spacing-large); min-height: 14rem; diff --git a/manager-dashboard/app/utils/common.tsx b/manager-dashboard/app/utils/common.tsx index 53338d34f..ea4f777fa 100644 --- a/manager-dashboard/app/utils/common.tsx +++ b/manager-dashboard/app/utils/common.tsx @@ -66,8 +66,9 @@ export const PROJECT_TYPE_FOOTPRINT = 2; export const PROJECT_TYPE_CHANGE_DETECTION = 3; export const PROJECT_TYPE_COMPLETENESS = 4; export const PROJECT_TYPE_STREET = 7; +export const PROJECT_TYPE_VALIDATE_IMAGE = 10; -export type ProjectType = 1 | 2 | 3 | 4 | 7; +export type ProjectType = 1 | 2 | 3 | 4 | 7 | 10; export const projectTypeLabelMap: { [key in ProjectType]: string @@ -77,6 +78,7 @@ export const projectTypeLabelMap: { [PROJECT_TYPE_CHANGE_DETECTION]: 'Compare', [PROJECT_TYPE_COMPLETENESS]: 'Completeness', [PROJECT_TYPE_STREET]: 'Street', + [PROJECT_TYPE_VALIDATE_IMAGE]: 'Validate Image', }; export type IconKey = 'add-outline' diff --git a/manager-dashboard/app/views/NewProject/ImageInput/index.tsx b/manager-dashboard/app/views/NewProject/ImageInput/index.tsx new file mode 100644 index 000000000..93f4dcde2 --- /dev/null +++ b/manager-dashboard/app/views/NewProject/ImageInput/index.tsx @@ -0,0 +1,73 @@ +import React from 'react'; + +import { + SetValueArg, + Error, + useFormObject, + getErrorObject, +} from '@togglecorp/toggle-form'; +import TextInput from '#components/TextInput'; + +import { + ImageType, +} from '../utils'; + +import styles from './styles.css'; + +const defaultImageValue: ImageType = { + sourceIdentifier: '', +}; + +interface Props { + value: ImageType; + onChange: (value: SetValueArg, index: number) => void | undefined; + index: number; + error: Error | undefined; + disabled?: boolean; + readOnly?: boolean; +} + +export default function ImageInput(props: Props) { + const { + value, + onChange, + index, + error: riskyError, + disabled, + readOnly, + } = props; + + const onImageChange = useFormObject(index, onChange, defaultImageValue); + + const error = getErrorObject(riskyError); + + return ( +
+ + + +
+ ); +} diff --git a/manager-dashboard/app/views/NewProject/ImageInput/styles.css b/manager-dashboard/app/views/NewProject/ImageInput/styles.css new file mode 100644 index 000000000..a6e6f1707 --- /dev/null +++ b/manager-dashboard/app/views/NewProject/ImageInput/styles.css @@ -0,0 +1,5 @@ +.image-input { + display: flex; + flex-direction: column; + gap: var(--spacing-medium); +} diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 811279723..23666c80b 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -3,6 +3,7 @@ import { _cs, isDefined, isNotDefined, + randomString, } from '@togglecorp/fujs'; import { useForm, @@ -10,6 +11,7 @@ import { createSubmitHandler, analyzeErrors, nonFieldError, + useFormArray, } from '@togglecorp/toggle-form'; import { getStorage, @@ -29,8 +31,14 @@ import { import { MdOutlinePublishedWithChanges, MdOutlineUnpublished, + MdAdd, } from 'react-icons/md'; +import { + IoIosTrash, +} from 'react-icons/io'; import { Link } from 'react-router-dom'; +import * as t from 'io-ts'; +import { isRight } from 'fp-ts/Either'; import UserContext from '#base/context/UserContext'; import projectTypeOptions from '#base/configs/projectTypes'; @@ -40,6 +48,7 @@ import TextInput from '#components/TextInput'; import NumberInput from '#components/NumberInput'; import SegmentInput from '#components/SegmentInput'; import GeoJsonFileInput from '#components/GeoJsonFileInput'; +import JsonFileInput from '#components/JsonFileInput'; import TileServerInput, { TILE_SERVER_BING, TILE_SERVER_ESRI, @@ -60,6 +69,7 @@ import { ProjectInputType, PROJECT_TYPE_BUILD_AREA, PROJECT_TYPE_FOOTPRINT, + PROJECT_TYPE_VALIDATE_IMAGE, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_STREET, @@ -73,6 +83,7 @@ import CustomOptionPreview from '#views/NewTutorial/CustomOptionInput/CustomOpti import { projectFormSchema, ProjectFormType, + ImageType, PartialProjectFormType, projectInputTypeOptions, filterOptions, @@ -84,12 +95,33 @@ import { getGroupSize, validateAoiOnOhsome, validateProjectIdOnHotTaskingManager, + MAX_IMAGES, } from './utils'; import BasicProjectInfoForm from './BasicProjectInfoForm'; +import ImageInput from './ImageInput'; // eslint-disable-next-line postcss-modules/no-unused-class import styles from './styles.css'; +const Image = t.type({ + id: t.number, + // width: t.number, + // height: t.number, + file_name: t.string, + // license: t.union([t.number, t.undefined]), + flickr_url: t.union([t.string, t.undefined]), + coco_url: t.union([t.string, t.undefined]), + // date_captured: DateFromISOString, +}); +const CocoDataset = t.type({ + // info: Info, + // licenses: t.array(License), + images: t.array(Image), + // annotations: t.array(Annotation), + // categories: t.array(Category) +}); +// type CocoDatasetType = t.TypeOf + const defaultProjectFormValue: PartialProjectFormType = { // projectType: PROJECT_TYPE_BUILD_AREA, projectNumber: 1, @@ -448,11 +480,77 @@ function NewProject(props: Props) { })), }))), [customOptionsFromValue]); - const optionsError = React.useMemo( + const customOptionsError = React.useMemo( () => getErrorObject(error?.customOptions), [error?.customOptions], ); + const { images } = value; + + const imagesError = React.useMemo( + () => getErrorObject(error?.images), + [error?.images], + ); + + const { + setValue: setImageValue, + removeValue: onImageRemove, + } = useFormArray< + 'images', + ImageType + >('images', setFieldValue); + + const handleCocoImport = React.useCallback( + (val) => { + const result = CocoDataset.decode(val); + if (!isRight(result)) { + // eslint-disable-next-line no-console + console.error('Invalid COCO format', result.left); + setError((err) => ({ + ...getErrorObject(err), + [nonFieldError]: 'Invalid COCO format', + })); + return; + } + if (result.right.images.length > MAX_IMAGES) { + setError((err) => ({ + ...getErrorObject(err), + [nonFieldError]: `Too many images ${result.right.images.length} uploaded. Please do not exceed ${MAX_IMAGES} images.`, + })); + return; + } + setFieldValue( + () => result.right.images.map((image) => ({ + sourceIdentifier: String(image.id), + fileName: image.file_name, + url: image.flickr_url || image.coco_url, + })), + 'images', + ); + }, + [setFieldValue, setError], + ); + + const handleAddImage = React.useCallback( + () => { + setFieldValue( + (oldValue: PartialProjectFormType['images']) => { + const safeOldValues = oldValue ?? []; + + const newDefineOption: ImageType = { + sourceIdentifier: randomString(), + }; + + return [...safeOldValues, newDefineOption]; + }, + 'images', + ); + }, + [ + setFieldValue, + ], + ); + // eslint-disable-next-line @typescript-eslint/no-empty-function const noOp = () => {}; @@ -492,8 +590,79 @@ function NewProject(props: Props) { disabled={submissionPending || projectTypeEmpty} /> + {(value.projectType === PROJECT_TYPE_VALIDATE_IMAGE) && ( + + + + )} + > + + {(images && images.length > 0) ? ( +
+ {images.map((image, index) => ( + + + + )} + > + + + ))} +
+ ) : ( + + name={undefined} + onChange={handleCocoImport} + disabled={ + submissionPending + || projectTypeEmpty + } + label="Import COCO file" + value={undefined} + /> + )} + + )} {( (value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_VALIDATE_IMAGE || value.projectType === PROJECT_TYPE_STREET) && customOptions && customOptions.length > 0 @@ -502,7 +671,7 @@ function NewProject(props: Props) { heading="Custom Options" > {(customOptions && customOptions.length > 0) ? (
@@ -517,7 +686,7 @@ function NewProject(props: Props) { value={option} index={index} onChange={noOp} - error={optionsError?.[option.value]} + error={customOptionsError?.[option.value]} readOnly /> @@ -743,7 +912,7 @@ function NewProject(props: Props) { value={value?.organizationId} onChange={setFieldValue} error={error?.organizationId} - label="Mapillary Organization ID" + label="Mapillary Organization IidD" hint="Provide a valid Mapillary organization ID to filter for images belonging to a specific organization. Empty indicates that no filter is set on organization." disabled={submissionPending || projectTypeEmpty} /> diff --git a/manager-dashboard/app/views/NewProject/styles.css b/manager-dashboard/app/views/NewProject/styles.css index cbfa76230..45aedf1bc 100644 --- a/manager-dashboard/app/views/NewProject/styles.css +++ b/manager-dashboard/app/views/NewProject/styles.css @@ -13,6 +13,14 @@ max-width: 70rem; gap: var(--spacing-large); + + .image-list { + display: flex; + flex-direction: column; + flex-grow: 1; + gap: var(--spacing-medium); + } + .custom-option-container { display: flex; gap: var(--spacing-large); diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index ce419e42d..be883e85d 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -34,6 +34,7 @@ import { ProjectInputType, PROJECT_TYPE_BUILD_AREA, PROJECT_TYPE_FOOTPRINT, + PROJECT_TYPE_VALIDATE_IMAGE, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_STREET, @@ -68,13 +69,14 @@ export interface ProjectFormType { projectImage: File; // image verificationNumber: number; groupSize: number; + maxTasksPerUser: number; + zoomLevel: number; geometry?: GeoJSON.GeoJSON | string; inputType?: ProjectInputType; TMId?: string; filter?: string; filterText?: string; - maxTasksPerUser: number; tileServer: TileServer; tileServerB?: TileServer; customOptions?: CustomOptionsForProject; @@ -87,6 +89,11 @@ export interface ProjectFormType { panoOnly?: boolean; isPano?: boolean | null; samplingThreshold?: number; + images?: { + sourceIdentifier: string; + fileName: string; + url: string; + }[]; } export const PROJECT_INPUT_TYPE_UPLOAD = 'aoi_file'; @@ -115,9 +122,11 @@ export const filterOptions = [ export type PartialProjectFormType = PartialForm< Omit & { projectImage?: File }, // NOTE: we do not want to change File and FeatureCollection to partials - 'geometry' | 'projectImage' | 'value' + 'geometry' | 'projectImage' | 'value' | 'sourceIdentifier' >; +export type ImageType = NonNullable[number]; + type ProjectFormSchema = ObjectSchema; type ProjectFormSchemaFields = ReturnType; @@ -127,6 +136,12 @@ type CustomOptionSchemaFields = ReturnType type CustomOptionFormSchema = ArraySchema; type CustomOptionFormSchemaMember = ReturnType; +type PartialImages = NonNullable[number]; +type ImageSchema = ObjectSchema; +type ImageSchemaFields = ReturnType +type ImageFormSchema = ArraySchema; +type ImageFormSchemaMember = ReturnType; + // FIXME: break this into multiple geometry conditions const DEFAULT_MAX_FEATURES = 20; // const DEFAULT_MAX_FEATURES = 10; @@ -194,6 +209,8 @@ function validGeometryCondition(zoomLevel: number | undefined | null) { return validGeometryConditionForZoom; } +export const MAX_IMAGES = 2000; + export const MAX_OPTIONS = 6; export const MIN_OPTIONS = 2; export const MAX_SUB_OPTIONS = 6; @@ -275,49 +292,16 @@ export const projectFormSchema: ProjectFormSchema = { lessThanOrEqualToCondition(250), ], }, - tileServer: { - fields: tileServerFieldsSchema, - }, maxTasksPerUser: { validations: [ integerCondition, greaterThanCondition(0), ], }, - dateRange: { - required: false, - }, - creatorId: { - required: false, - validations: [ - integerCondition, - greaterThanCondition(0), - ], - }, - organizationId: { - required: false, - validations: [ - integerCondition, - greaterThanCondition(0), - ], - }, - samplingThreshold: { - required: false, - validation: [ - greaterThanCondition(0), - ], - }, - panoOnly: { - required: false, - }, - isPano: { - required: false, - }, - randomizeOrder: { - required: false, - }, }; + // Common + baseSchema = addCondition( baseSchema, value, @@ -325,6 +309,7 @@ export const projectFormSchema: ProjectFormSchema = { ['customOptions'], (formValues) => { if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: { @@ -388,8 +373,8 @@ export const projectFormSchema: ProjectFormSchema = { const projectType = v?.projectType; if ( projectType === PROJECT_TYPE_BUILD_AREA - || projectType === PROJECT_TYPE_COMPLETENESS || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_COMPLETENESS ) { return { zoomLevel: { @@ -408,24 +393,6 @@ export const projectFormSchema: ProjectFormSchema = { }, ); - baseSchema = addCondition( - baseSchema, - value, - ['projectType'], - ['inputType'], - (v) => { - const projectType = v?.projectType; - if (projectType === PROJECT_TYPE_FOOTPRINT) { - return { - inputType: { required: true }, - }; - } - return { - inputType: { forceValue: nullValue }, - }; - }, - ); - baseSchema = addCondition( baseSchema, value, @@ -437,8 +404,8 @@ export const projectFormSchema: ProjectFormSchema = { const zoomLevel = v?.zoomLevel; if ( projectType === PROJECT_TYPE_BUILD_AREA - || projectType === PROJECT_TYPE_COMPLETENESS || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_COMPLETENESS || projectType === PROJECT_TYPE_STREET || (projectType === PROJECT_TYPE_FOOTPRINT && ( inputType === PROJECT_INPUT_TYPE_UPLOAD @@ -483,6 +450,51 @@ export const projectFormSchema: ProjectFormSchema = { }, ); + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['tileServer'], + (v) => { + const projectType = v?.projectType; + if ( + projectType === PROJECT_TYPE_BUILD_AREA + || projectType === PROJECT_TYPE_COMPLETENESS + || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_FOOTPRINT + ) { + return { + tileServer: { + fields: tileServerFieldsSchema, + }, + }; + } + return { + tileServer: { forceValue: nullValue }, + }; + }, + ); + + // Validate + + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['inputType'], + (v) => { + const projectType = v?.projectType; + if (projectType === PROJECT_TYPE_FOOTPRINT) { + return { + inputType: { required: true }, + }; + } + return { + inputType: { forceValue: nullValue }, + }; + }, + ); + baseSchema = addCondition( baseSchema, value, @@ -560,6 +572,103 @@ export const projectFormSchema: ProjectFormSchema = { }, ); + // Street + + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['dateRange', 'creatorId', 'organizationId', 'samplingThreshold', 'panoOnly', 'isPano', 'randomizeOrder'], + (formValues) => { + if (formValues?.projectType === PROJECT_TYPE_STREET) { + return { + dateRange: { + required: false, + }, + creatorId: { + required: false, + validations: [ + integerCondition, + greaterThanCondition(0), + ], + }, + organizationId: { + required: false, + validations: [ + integerCondition, + greaterThanCondition(0), + ], + }, + samplingThreshold: { + required: false, + validations: [ + greaterThanCondition(0), + ], + }, + panoOnly: { + required: false, + }, + // FIXME: This is not used. + isPano: { + required: false, + }, + randomizeOrder: { + required: false, + }, + }; + } + return { + dateRange: { forceValue: nullValue }, + creatorId: { forceValue: nullValue }, + organizationId: { forceValue: nullValue }, + samplingThreshold: { forceValue: nullValue }, + panoOnly: { forceValue: nullValue }, + isPano: { forceValude: nullValue }, + randomizeOrder: { forceValue: nullValue }, + }; + }, + ); + + // Validate Image + + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['images'], + (formValues) => { + // FIXME: Add "unique" constraint for sourceIdentifier and fileName + // FIXME: Add max length constraint + if (formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { + return { + images: { + keySelector: (key) => key.sourceIdentifier, + member: (): ImageFormSchemaMember => ({ + fields: (): ImageSchemaFields => ({ + sourceIdentifier: { + required: true, + requiredValidation: requiredStringCondition, + }, + fileName: { + required: true, + requiredValidation: requiredStringCondition, + }, + url: { + required: true, + requiredValidation: requiredStringCondition, + validations: [urlCondition], + }, + }), + }), + }, + }; + } + return { + images: { forceValue: nullValue }, + }; + }, + ); + return baseSchema; }, }; @@ -588,6 +697,7 @@ export function getGroupSize(projectType: ProjectType | undefined) { } if (projectType === PROJECT_TYPE_FOOTPRINT + || projectType === PROJECT_TYPE_VALIDATE_IMAGE || projectType === PROJECT_TYPE_CHANGE_DETECTION || projectType === PROJECT_TYPE_STREET) { return 25; diff --git a/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx b/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx new file mode 100644 index 000000000..f985d2ed8 --- /dev/null +++ b/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx @@ -0,0 +1,92 @@ +import React from 'react'; + +import { + SetValueArg, + Error, + useFormObject, + getErrorObject, +} from '@togglecorp/toggle-form'; +import TextInput from '#components/TextInput'; +import NumberInput from '#components/NumberInput'; + +import { + ImageType, +} from '../utils'; + +import styles from './styles.css'; + +const defaultImageValue: ImageType = { + sourceIdentifier: '', +}; + +interface Props { + value: ImageType; + onChange: (value: SetValueArg, index: number) => void | undefined; + index: number; + error: Error | undefined; + disabled?: boolean; + readOnly?: boolean; +} + +export default function ImageInput(props: Props) { + const { + value, + onChange, + index, + error: riskyError, + disabled, + readOnly, + } = props; + + const onImageChange = useFormObject(index, onChange, defaultImageValue); + + const error = getErrorObject(riskyError); + + return ( +
+ + + + + {/* FIXME: Use select input */} + +
+ ); +} diff --git a/manager-dashboard/app/views/NewTutorial/ImageInput/styles.css b/manager-dashboard/app/views/NewTutorial/ImageInput/styles.css new file mode 100644 index 000000000..a6e6f1707 --- /dev/null +++ b/manager-dashboard/app/views/NewTutorial/ImageInput/styles.css @@ -0,0 +1,5 @@ +.image-input { + display: flex; + flex-direction: column; + gap: var(--spacing-medium); +} diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/FootprintGeoJsonPreview/index.tsx b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/FootprintGeoJsonPreview/index.tsx index f381ff4f9..2ab9cbe36 100644 --- a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/FootprintGeoJsonPreview/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/FootprintGeoJsonPreview/index.tsx @@ -15,7 +15,7 @@ import { import styles from './styles.css'; // NOTE: the padding is selected wrt the size of the preview -const footprintGeojsonPadding = [140, 140]; +const footprintGeojsonPadding: [number, number] = [140, 140]; interface Props { className?: string; diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/index.tsx b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/index.tsx new file mode 100644 index 000000000..3dfa8fb98 --- /dev/null +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/index.tsx @@ -0,0 +1,81 @@ +import React from 'react'; +import { _cs } from '@togglecorp/fujs'; + +import MobilePreview from '#components/MobilePreview'; +import { IconKey, iconMap } from '#utils/common'; + +import { + ImageType, + colorKeyToColorMap, + PartialCustomOptionsType, +} from '../../utils'; +import styles from './styles.css'; + +interface Props { + className?: string; + image?: ImageType; + previewPopUp?: { + title?: string; + description?: string; + icon?: IconKey; + } + customOptions: PartialCustomOptionsType | undefined; + lookFor: string | undefined; +} + +export default function ValidateImagePreview(props: Props) { + const { + className, + previewPopUp, + customOptions, + lookFor, + image, + } = props; + + const Comp = previewPopUp?.icon ? iconMap[previewPopUp.icon] : undefined; + + return ( + } + popupTitle={previewPopUp?.title || '{title}'} + popupDescription={previewPopUp?.description || '{description}'} + > + Preview +
+ {customOptions?.map((option) => { + const Icon = option.icon + ? iconMap[option.icon] + : iconMap['flag-outline']; + return ( +
+
+ {Icon && ( + + )} +
+
+ ); + })} +
+ + ); +} diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css new file mode 100644 index 000000000..d3642b14d --- /dev/null +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css @@ -0,0 +1,37 @@ +.validate-image-preview { + .content { + display: flex; + flex-direction: column; + gap: var(--spacing-large); + + .image-preview { + position: relative; + border: 1px solid red; + width: 100%; + height: var(--height-mobile-preview-validate-image-content); + } + + .options { + display: grid; + flex-grow: 1; + grid-template-columns: 1fr 1fr 1fr; + grid-gap: var(--spacing-large); + + .option-container { + display: flex; + align-items: center; + justify-content: center; + + .option { + display: flex; + align-items: center; + justify-content: center; + border-radius: 50%; + width: 2.5rem; + height: 2.5rem; + font-size: var(--font-size-extra-large); + } + } + } + } +} diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx index b309be7ff..2ba9d05fe 100644 --- a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx @@ -18,6 +18,7 @@ import { PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_STREET, + PROJECT_TYPE_VALIDATE_IMAGE, } from '#utils/common'; import TextInput from '#components/TextInput'; import Heading from '#components/Heading'; @@ -25,6 +26,7 @@ import SelectInput from '#components/SelectInput'; import SegmentInput from '#components/SegmentInput'; import { + ImageType, TutorialTasksGeoJSON, FootprintGeoJSON, BuildAreaGeoJSON, @@ -34,6 +36,7 @@ import { import BuildAreaGeoJsonPreview from './BuildAreaGeoJsonPreview'; import FootprintGeoJsonPreview from './FootprintGeoJsonPreview'; import ChangeDetectionGeoJsonPreview from './ChangeDetectionGeoJsonPreview'; +import ValidateImagePreview from './ValidateImagePreview'; import styles from './styles.css'; type ScenarioType = { @@ -78,6 +81,7 @@ interface Props { index: number, error: Error | undefined; geoJson: TutorialTasksGeoJSON | undefined; + images: ImageType[] | undefined; projectType: ProjectType | undefined; urlA: string | undefined; urlB: string | undefined; @@ -94,6 +98,7 @@ export default function ScenarioPageInput(props: Props) { index, error: riskyError, geoJson: geoJsonFromProps, + images, urlA, projectType, urlB, @@ -171,7 +176,21 @@ export default function ScenarioPageInput(props: Props) { [geoJsonFromProps, scenarioId], ); - const activeSegmentInput: ScenarioSegmentType['value'] = projectType && projectType !== PROJECT_TYPE_FOOTPRINT + const image = React.useMemo( + () => { + if (!images) { + return undefined; + } + return images.find((img) => img.screen === scenarioId); + }, + [images, scenarioId], + ); + + const activeSegmentInput: ScenarioSegmentType['value'] = ( + projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_VALIDATE_IMAGE + ) ? activeSegmentInputFromState : 'instructions'; @@ -214,7 +233,11 @@ export default function ScenarioPageInput(props: Props) { disabled={disabled} />
- {projectType && projectType !== PROJECT_TYPE_FOOTPRINT && ( + {( + projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_VALIDATE_IMAGE + ) && ( <> Hint @@ -252,7 +275,11 @@ export default function ScenarioPageInput(props: Props) { )} - {projectType && projectType !== PROJECT_TYPE_FOOTPRINT && ( + {( + projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_VALIDATE_IMAGE + ) && ( <> Success @@ -319,6 +346,14 @@ export default function ScenarioPageInput(props: Props) { lookFor={lookFor} /> )} + {projectType === PROJECT_TYPE_VALIDATE_IMAGE && ( + + )} {projectType === PROJECT_TYPE_STREET && (
Preview not available. @@ -326,6 +361,7 @@ export default function ScenarioPageInput(props: Props) { )} {(projectType && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_VALIDATE_IMAGE && projectType !== PROJECT_TYPE_STREET) && ( + export function getDuplicates( list: T[], keySelector: (item: T) => K, @@ -157,6 +185,7 @@ function getGeoJSONError( return 'GeoJSON does not contain iterable features'; } + // FIXME: Use io-ts // Check properties schema const projectSchemas: { [key in ProjectType]: Record; @@ -195,6 +224,9 @@ function getGeoJSONError( reference: 'number', screen: 'number', }, + [PROJECT_TYPE_VALIDATE_IMAGE]: { + // NOTE: We do not use geojson import for validate image project + }, }; const schemaErrors = tutorialTasks.features.map( (feature) => checkSchema( @@ -406,6 +438,14 @@ function NewTutorial(props: Props) { InformationPagesType >('informationPages', setFieldValue); + const { + setValue: setImageValue, + // removeValue: onImageRemove, + } = useFormArray< + 'images', + ImageType + >('images', setFieldValue); + const handleSubmission = React.useCallback(( finalValuesFromProps: PartialTutorialFormType, ) => { @@ -600,7 +640,6 @@ function NewTutorial(props: Props) { })); return; } - setFieldValue(tutorialTasks, 'tutorialTasks'); const uniqueArray = unique( @@ -616,7 +655,6 @@ function NewTutorial(props: Props) { success: {}, } )); - setFieldValue(tutorialTaskArray, 'scenarioPages'); }, [setFieldValue, setError, value?.projectType]); @@ -645,6 +683,56 @@ function NewTutorial(props: Props) { [setFieldValue], ); + const handleCocoImport = React.useCallback( + (val) => { + const result = CocoDataset.decode(val); + if (!isRight(result)) { + // eslint-disable-next-line no-console + console.error('Invalid COCO format', result.left); + setError((err) => ({ + ...getErrorObject(err), + [nonFieldError]: 'Invalid COCO format', + })); + return; + } + if (result.right.images.length > MAX_IMAGES) { + setError((err) => ({ + ...getErrorObject(err), + [nonFieldError]: `Too many images ${result.right.images.length} uploaded. Please do not exceed ${MAX_IMAGES} images.`, + })); + return; + } + + const newImages = result.right.images.map((image, index) => ({ + sourceIdentifier: String(image.id), + fileName: image.file_name, + url: image.flickr_url || image.coco_url, + screen: index + 1, + referenceAnswer: 1, + })); + setFieldValue( + () => newImages, + 'images', + ); + + const uniqueArray = unique( + newImages, + ((img) => img.screen), + ); + const sorted = uniqueArray.sort((a, b) => a.screen - b.screen); + const tutorialTaskArray = sorted?.map((img) => ( + { + scenarioId: img.screen, + hint: {}, + instructions: {}, + success: {}, + } + )); + setFieldValue(tutorialTaskArray, 'scenarioPages'); + }, + [setFieldValue, setError], + ); + const submissionPending = ( tutorialSubmissionStatus === 'started' || tutorialSubmissionStatus === 'imageUpload' @@ -678,6 +766,11 @@ function NewTutorial(props: Props) { [error?.informationPages], ); + const imagesError = React.useMemo( + () => getErrorObject(error?.images), + [error?.images], + ); + const hasErrors = React.useMemo( () => analyzeErrors(error), [error], @@ -693,6 +786,8 @@ function NewTutorial(props: Props) { ...options, ...subOptions, ].filter(isDefined); + + // FIXME: Add warning here for validate image return getGeoJSONWarning( value?.tutorialTasks, value?.projectType, @@ -719,6 +814,7 @@ function NewTutorial(props: Props) { const { customOptions, informationPages, + images, } = value; const handleProjectTypeChange = React.useCallback( @@ -774,6 +870,7 @@ function NewTutorial(props: Props) { {( value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_VALIDATE_IMAGE || value.projectType === PROJECT_TYPE_STREET ) && ( ) } + {value.projectType === PROJECT_TYPE_VALIDATE_IMAGE && ( + + + + name={undefined} + onChange={handleCocoImport} + disabled={ + submissionPending + || projectTypeEmpty + } + label="Import COCO file" + value={undefined} + /> + {(images && images.length > 0) ? ( +
+ {images.map((image, index) => ( + + + + ))} +
+ ) : ( + + )} + + )} - + {value?.projectType !== PROJECT_TYPE_VALIDATE_IMAGE && ( + + )}
{value.scenarioPages?.map((task, index) => ( ))} {(value.scenarioPages?.length ?? 0) === 0 && ( - + <> + {value.projectType !== PROJECT_TYPE_VALIDATE_IMAGE ? ( + + ) : ( + + )} + )}
diff --git a/manager-dashboard/app/views/NewTutorial/styles.css b/manager-dashboard/app/views/NewTutorial/styles.css index 7242f2344..40b8cbdf6 100644 --- a/manager-dashboard/app/views/NewTutorial/styles.css +++ b/manager-dashboard/app/views/NewTutorial/styles.css @@ -20,6 +20,13 @@ gap: var(--spacing-medium); } + .image-list { + display: flex; + flex-direction: column; + flex-grow: 1; + gap: var(--spacing-medium); + } + .custom-option-container { display: flex; gap: var(--spacing-large); diff --git a/manager-dashboard/app/views/NewTutorial/utils.ts b/manager-dashboard/app/views/NewTutorial/utils.ts index 67f5e4af5..74fc7711b 100644 --- a/manager-dashboard/app/views/NewTutorial/utils.ts +++ b/manager-dashboard/app/views/NewTutorial/utils.ts @@ -8,6 +8,7 @@ import { nullValue, ArraySchema, addCondition, + urlCondition, } from '@togglecorp/toggle-form'; import { isDefined, @@ -27,6 +28,7 @@ import { PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_STREET, + PROJECT_TYPE_VALIDATE_IMAGE, IconKey, } from '#utils/common'; @@ -285,6 +287,36 @@ export const defaultStreetCustomOptions: PartialTutorialFormType['customOptions' }, ]; +export const defaultValidateImageCustomOptions: PartialTutorialFormType['customOptions'] = [ + { + optionId: 1, + value: 1, + title: 'Yes', + icon: 'checkmark-outline', + iconColor: colorKeyToColorMap.green, + // FIXME: Add description + description: 'Yes', + }, + { + optionId: 2, + value: 0, + title: 'No', + icon: 'close-outline', + iconColor: colorKeyToColorMap.red, + // FIXME: Add description + description: 'No', + }, + { + optionId: 3, + value: 2, + title: 'Not Sure', + icon: 'remove-outline', + iconColor: colorKeyToColorMap.gray, + // FIXME: Add description + description: 'Not Sure', + }, +]; + export function deleteKey( value: T, key: K, @@ -305,6 +337,10 @@ export function getDefaultOptions(projectType: ProjectType | undefined) { return defaultStreetCustomOptions; } + if (projectType === PROJECT_TYPE_VALIDATE_IMAGE) { + return defaultValidateImageCustomOptions; + } + return undefined; } @@ -426,7 +462,6 @@ export interface TutorialFormType { title: string; }; }[]; - tutorialTasks?: TutorialTasksGeoJSON, exampleImage1: File; exampleImage2: File; projectType: ProjectType; @@ -434,6 +469,15 @@ export interface TutorialFormType { zoomLevel?: number; customOptions?: CustomOptions; informationPages: InformationPages; + + tutorialTasks?: TutorialTasksGeoJSON, + images?: { + sourceIdentifier: string; + fileName: string; + url: string; + referenceAnswer: number; + screen: number; + }[]; } export type PartialTutorialFormType = PartialForm< @@ -442,9 +486,11 @@ export type PartialTutorialFormType = PartialForm< exampleImage2?: File; }, // NOTE: we do not want to change File and FeatureCollection to partials - 'image' | 'tutorialTasks' | 'exampleImage1' | 'exampleImage2' | 'scenarioId' | 'optionId' | 'subOptionsId' | 'pageNumber' | 'blockNumber' | 'blockType' | 'imageFile' + 'image' | 'tutorialTasks' | 'exampleImage1' | 'exampleImage2' | 'scenarioId' | 'optionId' | 'subOptionsId' | 'pageNumber' | 'blockNumber' | 'blockType' | 'imageFile' | 'sourceIdentifier' >; +export type ImageType = NonNullable[number]; + type TutorialFormSchema = ObjectSchema; type TutorialFormSchemaFields = ReturnType; @@ -462,6 +508,12 @@ export type CustomOptionSchemaFields = ReturnType export type CustomOptionFormSchema = ArraySchema; export type CustomOptionFormSchemaMember = ReturnType; +type PartialImages = NonNullable[number]; +type ImageSchema = ObjectSchema; +type ImageSchemaFields = ReturnType +type ImageFormSchema = ArraySchema; +type ImageFormSchemaMember = ReturnType; + export type InformationPagesType = NonNullable[number] type InformationPagesSchema = ObjectSchema; type InformationPagesSchemaFields = ReturnType @@ -473,6 +525,8 @@ export type PartialInformationPagesType = PartialTutorialFormType['informationPa export type PartialCustomOptionsType = PartialTutorialFormType['customOptions']; export type PartialBlocksType = NonNullable[number]>['blocks']; +export const MAX_IMAGES = 20; + export const MAX_OPTIONS = 6; export const MIN_OPTIONS = 2; export const MAX_SUB_OPTIONS = 6; @@ -500,12 +554,6 @@ export const tutorialFormSchema: TutorialFormSchema = { requiredValidation: requiredStringCondition, validations: [getNoMoreThanNCharacterCondition(MD_TEXT_MAX_LENGTH)], }, - tileServer: { - fields: tileServerFieldsSchema, - }, - tutorialTasks: { - required: true, - }, informationPages: { validation: (info) => { if (info && info.length > MAX_INFO_PAGES) { @@ -564,6 +612,8 @@ export const tutorialFormSchema: TutorialFormSchema = { }, }; + // common + baseSchema = addCondition( baseSchema, value, @@ -601,7 +651,11 @@ export const tutorialFormSchema: TutorialFormSchema = { }), }, }; - if (projectType && projectType !== PROJECT_TYPE_FOOTPRINT) { + if ( + projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_VALIDATE_IMAGE + ) { fields = { ...fields, hint: { @@ -776,6 +830,7 @@ export const tutorialFormSchema: TutorialFormSchema = { }; if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: customOptionField, @@ -809,6 +864,23 @@ export const tutorialFormSchema: TutorialFormSchema = { }), ); + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['tileServer'], + (v) => ( + v?.projectType !== PROJECT_TYPE_VALIDATE_IMAGE + ? { + tileServer: { + fields: tileServerFieldsSchema, + }, + } : { + tileServer: { forceValue: nullValue }, + } + ), + ); + baseSchema = addCondition( baseSchema, value, @@ -824,6 +896,72 @@ export const tutorialFormSchema: TutorialFormSchema = { tileServerB: { forceValue: nullValue }, }), ); + + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['tutorialTasks'], + (formValues) => { + if (formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { + return { + tutorialTasks: { forceValue: nullValue }, + }; + } + return { + tutorialTasks: { + required: true, + }, + }; + }, + ); + + // validate image + + baseSchema = addCondition( + baseSchema, + value, + ['projectType'], + ['images'], + (formValues) => { + // FIXME: Add "unique" constraint for sourceIdentifier and fileName + // FIXME: Add max length constraint + if (formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { + return { + images: { + keySelector: (key) => key.sourceIdentifier, + member: (): ImageFormSchemaMember => ({ + fields: (): ImageSchemaFields => ({ + sourceIdentifier: { + required: true, + requiredValidation: requiredStringCondition, + }, + fileName: { + required: true, + requiredValidation: requiredStringCondition, + }, + url: { + required: true, + requiredValidation: requiredStringCondition, + validations: [urlCondition], + }, + referenceAnswer: { + required: true, + }, + screen: { + required: true, + }, + }), + }), + }, + }; + } + return { + images: { forceValue: nullValue }, + }; + }, + ); + return baseSchema; }, }; diff --git a/manager-dashboard/package.json b/manager-dashboard/package.json index de3c020bd..4b0ee3c1a 100644 --- a/manager-dashboard/package.json +++ b/manager-dashboard/package.json @@ -44,8 +44,11 @@ "apollo-upload-client": "^16.0.0", "core-js": "3", "firebase": "^9.9.0", + "fp-ts": "^2.16.10", "graphql": "^15.5.1", "graphql-anywhere": "^4.2.7", + "io-ts": "^2.2.22", + "io-ts-types": "^0.5.19", "leaflet": "^1.8.0", "react": "^17.0.2", "react-dom": "^17.0.2", diff --git a/manager-dashboard/yarn.lock b/manager-dashboard/yarn.lock index a3597eea5..66718bef1 100644 --- a/manager-dashboard/yarn.lock +++ b/manager-dashboard/yarn.lock @@ -6668,6 +6668,11 @@ forwarded@0.2.0: resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== +fp-ts@^2.16.10: + version "2.16.10" + resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.16.10.tgz#829b82a46571c2dc202bed38a9c2eeec603e38c4" + integrity sha512-vuROzbNVfCmUkZSUbnWSltR1sbheyQbTzug7LB/46fEa1c0EucLeBaCEUE0gF3ZGUGBt9lVUiziGOhhj6K1ORA== + fraction.js@^4.1.1: version "4.1.2" resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.1.2.tgz#13e420a92422b6cf244dff8690ed89401029fbe8" @@ -7480,6 +7485,16 @@ invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +io-ts-types@^0.5.19: + version "0.5.19" + resolved "https://registry.yarnpkg.com/io-ts-types/-/io-ts-types-0.5.19.tgz#9c04fa73f15992436605218a5686b610efa7a5d3" + integrity sha512-kQOYYDZG5vKre+INIDZbLeDJe+oM+4zLpUkjXyTMyUfoCpjJNyi29ZLkuEAwcPufaYo3yu/BsemZtbdD+NtRfQ== + +io-ts@^2.2.22: + version "2.2.22" + resolved "https://registry.yarnpkg.com/io-ts/-/io-ts-2.2.22.tgz#5ab0d3636fe8494a275f0266461ab019da4b8d0b" + integrity sha512-FHCCztTkHoV9mdBsHpocLpdTAfh956ZQcIkWQxxS0U5HT53vtrcuYdQneEJKH6xILaLNzXVl2Cvwtoy8XNN0AA== + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" From fa13b4fb56cd19dc6cf92ce65c5d5355c93dfe20 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 27 Jun 2025 17:15:34 +0545 Subject: [PATCH 30/35] feat(validate_image): validations for coco file - create a input for coco file - add validations while importing coco file - add max validation for images - always show coco import on create project - add validation on forms for no. of images - add a select input for option selection in image reference - add warning if undefined option used in image reference --- .../app/components/CocoFileInput/index.tsx | 80 ++++++++++++++ .../app/components/JsonFileInput/index.tsx | 2 +- .../app/views/NewProject/index.tsx | 73 +++++-------- .../app/views/NewProject/utils.ts | 7 +- .../views/NewTutorial/ImageInput/index.tsx | 49 ++++++++- .../ValidateImagePreview/styles.css | 1 - .../app/views/NewTutorial/index.tsx | 100 +++++++++--------- .../app/views/NewTutorial/utils.ts | 7 +- 8 files changed, 214 insertions(+), 105 deletions(-) create mode 100644 manager-dashboard/app/components/CocoFileInput/index.tsx diff --git a/manager-dashboard/app/components/CocoFileInput/index.tsx b/manager-dashboard/app/components/CocoFileInput/index.tsx new file mode 100644 index 000000000..125e0c592 --- /dev/null +++ b/manager-dashboard/app/components/CocoFileInput/index.tsx @@ -0,0 +1,80 @@ +import React from 'react'; +import * as t from 'io-ts'; +import { isRight } from 'fp-ts/Either'; + +import JsonFileInput, { Props as JsonFileInputProps } from '#components/JsonFileInput'; + +const Image = t.type({ + id: t.number, + // width: t.number, + // height: t.number, + file_name: t.string, + // license: t.union([t.number, t.undefined]), + flickr_url: t.union([t.string, t.undefined]), + coco_url: t.union([t.string, t.undefined]), + // date_captured: DateFromISOString, +}); + +const CocoDataset = t.type({ + // info: Info, + // licenses: t.array(License), + images: t.array(Image), + // annotations: t.array(Annotation), + // categories: t.array(Category) +}); +export type CocoDatasetType = t.TypeOf + +interface Props extends Omit, 'onChange' | 'value'> { + value: CocoDatasetType | undefined; + maxLength: number; + onChange: (newValue: CocoDatasetType | undefined, name: N) => void; +} +function CocoFileInput(props: Props) { + const { + name, + onChange, + error, + maxLength, + ...otherProps + } = props; + + const [ + internalErrorMessage, + setInternalErrorMessage, + ] = React.useState(); + + const handleChange = React.useCallback( + (val) => { + const result = CocoDataset.decode(val); + if (!isRight(result)) { + // eslint-disable-next-line no-console + console.error('Invalid COCO format', result.left); + setInternalErrorMessage('Invalid COCO format'); + return; + } + if (result.right.images.length > maxLength) { + setInternalErrorMessage(`Too many images ${result.right.images.length} uploaded. Please do not exceed ${maxLength} images.`); + return; + } + const uniqueIdentifiers = new Set(result.right.images.map((item) => item.id)); + if (uniqueIdentifiers.size < result.right.images.length) { + setInternalErrorMessage('Each image should have a unique id.'); + return; + } + setInternalErrorMessage(undefined); + onChange(result.right, name); + }, + [onChange, maxLength, name], + ); + + return ( + + ); +} + +export default CocoFileInput; diff --git a/manager-dashboard/app/components/JsonFileInput/index.tsx b/manager-dashboard/app/components/JsonFileInput/index.tsx index bda27a599..023abde95 100644 --- a/manager-dashboard/app/components/JsonFileInput/index.tsx +++ b/manager-dashboard/app/components/JsonFileInput/index.tsx @@ -23,7 +23,7 @@ function readUploadedFileAsText(inputFile: File) { const ONE_MB = 1024 * 1024; const DEFAULT_MAX_FILE_SIZE = ONE_MB; -interface Props extends Omit, 'value' | 'onChange' | 'accept'> { +export interface Props extends Omit, 'value' | 'onChange' | 'accept'> { maxFileSize?: number; value: T | undefined | null; onChange: (newValue: T | undefined, name: N) => void; diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 23666c80b..cbc94b5ad 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -37,8 +37,6 @@ import { IoIosTrash, } from 'react-icons/io'; import { Link } from 'react-router-dom'; -import * as t from 'io-ts'; -import { isRight } from 'fp-ts/Either'; import UserContext from '#base/context/UserContext'; import projectTypeOptions from '#base/configs/projectTypes'; @@ -48,7 +46,7 @@ import TextInput from '#components/TextInput'; import NumberInput from '#components/NumberInput'; import SegmentInput from '#components/SegmentInput'; import GeoJsonFileInput from '#components/GeoJsonFileInput'; -import JsonFileInput from '#components/JsonFileInput'; +import CocoFileInput, { CocoDatasetType } from '#components/CocoFileInput'; import TileServerInput, { TILE_SERVER_BING, TILE_SERVER_ESRI, @@ -57,6 +55,7 @@ import TileServerInput, { import InputSection from '#components/InputSection'; import Button from '#components/Button'; import NonFieldError from '#components/NonFieldError'; +import EmptyMessage from '#components/EmptyMessage'; import AnimatedSwipeIcon from '#components/AnimatedSwipeIcon'; import ExpandableContainer from '#components/ExpandableContainer'; import AlertBanner from '#components/AlertBanner'; @@ -103,25 +102,6 @@ import ImageInput from './ImageInput'; // eslint-disable-next-line postcss-modules/no-unused-class import styles from './styles.css'; -const Image = t.type({ - id: t.number, - // width: t.number, - // height: t.number, - file_name: t.string, - // license: t.union([t.number, t.undefined]), - flickr_url: t.union([t.string, t.undefined]), - coco_url: t.union([t.string, t.undefined]), - // date_captured: DateFromISOString, -}); -const CocoDataset = t.type({ - // info: Info, - // licenses: t.array(License), - images: t.array(Image), - // annotations: t.array(Annotation), - // categories: t.array(Category) -}); -// type CocoDatasetType = t.TypeOf - const defaultProjectFormValue: PartialProjectFormType = { // projectType: PROJECT_TYPE_BUILD_AREA, projectNumber: 1, @@ -501,26 +481,16 @@ function NewProject(props: Props) { >('images', setFieldValue); const handleCocoImport = React.useCallback( - (val) => { - const result = CocoDataset.decode(val); - if (!isRight(result)) { - // eslint-disable-next-line no-console - console.error('Invalid COCO format', result.left); - setError((err) => ({ - ...getErrorObject(err), - [nonFieldError]: 'Invalid COCO format', - })); - return; - } - if (result.right.images.length > MAX_IMAGES) { - setError((err) => ({ - ...getErrorObject(err), - [nonFieldError]: `Too many images ${result.right.images.length} uploaded. Please do not exceed ${MAX_IMAGES} images.`, - })); + (val: CocoDatasetType | undefined) => { + if (isNotDefined(val)) { + setFieldValue( + [], + 'images', + ); return; } setFieldValue( - () => result.right.images.map((image) => ({ + () => val.images.map((image) => ({ sourceIdentifier: String(image.id), fileName: image.file_name, url: image.flickr_url || image.coco_url, @@ -528,7 +498,7 @@ function NewProject(props: Props) { 'images', ); }, - [setFieldValue, setError], + [setFieldValue], ); const handleAddImage = React.useCallback( @@ -613,6 +583,17 @@ function NewProject(props: Props) { + {(images && images.length > 0) ? (
{images.map((image, index) => ( @@ -647,15 +628,9 @@ function NewProject(props: Props) { ))}
) : ( - - name={undefined} - onChange={handleCocoImport} - disabled={ - submissionPending - || projectTypeEmpty - } - label="Import COCO file" - value={undefined} + )} diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index be883e85d..e2ac2731f 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -638,11 +638,16 @@ export const projectFormSchema: ProjectFormSchema = { ['images'], (formValues) => { // FIXME: Add "unique" constraint for sourceIdentifier and fileName - // FIXME: Add max length constraint if (formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { return { images: { keySelector: (key) => key.sourceIdentifier, + validation: (values) => { + if (values && values.length > MAX_IMAGES) { + return `Too many images ${values.length}. Please do not exceed ${MAX_IMAGES} images.`; + } + return undefined; + }, member: (): ImageFormSchemaMember => ({ fields: (): ImageSchemaFields => ({ sourceIdentifier: { diff --git a/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx b/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx index f985d2ed8..ca10b5806 100644 --- a/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/ImageInput/index.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { SetValueArg, @@ -6,11 +6,14 @@ import { useFormObject, getErrorObject, } from '@togglecorp/toggle-form'; +import { isNotDefined, isDefined, unique } from '@togglecorp/fujs'; import TextInput from '#components/TextInput'; +import SelectInput from '#components/SelectInput'; import NumberInput from '#components/NumberInput'; import { ImageType, + PartialCustomOptionsType, } from '../utils'; import styles from './styles.css'; @@ -26,6 +29,7 @@ interface Props { error: Error | undefined; disabled?: boolean; readOnly?: boolean; + customOptions: PartialCustomOptionsType | undefined; } export default function ImageInput(props: Props) { @@ -36,8 +40,45 @@ export default function ImageInput(props: Props) { error: riskyError, disabled, readOnly, + customOptions, } = props; + const flattenedOptions = useMemo( + () => { + const opts = customOptions?.flatMap( + (option) => ([ + { + key: option.value, + label: option.title, + }, + ...(option.subOptions ?? []).map( + (subOption) => ({ + key: subOption.value, + label: subOption.description, + }), + ), + ]), + ) ?? []; + + const validOpts = opts.map( + (option) => { + if (isNotDefined(option.key)) { + return undefined; + } + return { + ...option, + key: option.key, + }; + }, + ).filter(isDefined); + return unique( + validOpts, + (option) => option.key, + ); + }, + [customOptions], + ); + const onImageChange = useFormObject(index, onChange, defaultImageValue); const error = getErrorObject(riskyError); @@ -78,12 +119,14 @@ export default function ImageInput(props: Props) { disabled={disabled} readOnly /> - {/* FIXME: Use select input */} - option.key} + labelSelector={(option) => option.label ?? `Option ${option.key}`} + options={flattenedOptions} error={error?.referenceAnswer} disabled={disabled || readOnly} /> diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css index d3642b14d..5f708d4a5 100644 --- a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/ValidateImagePreview/styles.css @@ -6,7 +6,6 @@ .image-preview { position: relative; - border: 1px solid red; width: 100%; height: var(--height-mobile-preview-validate-image-content); } diff --git a/manager-dashboard/app/views/NewTutorial/index.tsx b/manager-dashboard/app/views/NewTutorial/index.tsx index 1830402a0..162361ddd 100644 --- a/manager-dashboard/app/views/NewTutorial/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/index.tsx @@ -14,7 +14,6 @@ import { createSubmitHandler, analyzeErrors, useFormArray, - nonFieldError, } from '@togglecorp/toggle-form'; import { getStorage, @@ -41,8 +40,6 @@ import { IoInformationCircleOutline, } from 'react-icons/io5'; import { Link } from 'react-router-dom'; -import * as t from 'io-ts'; -import { isRight } from 'fp-ts/Either'; import UserContext from '#base/context/UserContext'; import projectTypeOptions from '#base/configs/projectTypes'; @@ -53,7 +50,7 @@ import NumberInput from '#components/NumberInput'; import Heading from '#components/Heading'; import SegmentInput from '#components/SegmentInput'; import GeoJsonFileInput from '#components/GeoJsonFileInput'; -import JsonFileInput from '#components/JsonFileInput'; +import CocoFileInput, { CocoDatasetType } from '#components/CocoFileInput'; import ExpandableContainer from '#components/ExpandableContainer'; import PopupButton from '#components/PopupButton'; import TileServerInput, { @@ -109,26 +106,6 @@ import InformationPageInput from './InformationPageInput'; import ImageInput from './ImageInput'; import styles from './styles.css'; -// FIXME: let's not duplicate this logic -const Image = t.type({ - id: t.number, - // width: t.number, - // height: t.number, - file_name: t.string, - // license: t.union([t.number, t.undefined]), - flickr_url: t.union([t.string, t.undefined]), - coco_url: t.union([t.string, t.undefined]), - // date_captured: DateFromISOString, -}); -const CocoDataset = t.type({ - // info: Info, - // licenses: t.array(License), - images: t.array(Image), - // annotations: t.array(Annotation), - // categories: t.array(Category) -}); -// type CocoDatasetType = t.TypeOf - export function getDuplicates( list: T[], keySelector: (item: T) => K, @@ -351,6 +328,27 @@ function getGeoJSONWarning( return errors; } +function getImagesWarning( + images: ImageType[], + customOptions: number[], +) { + const errors = []; + + const usedValues = images.map((item) => item.referenceAnswer).filter(isDefined); + + const usedValuesSet = new Set(usedValues); + const customOptionsSet = new Set(customOptions); + + const invalidUsedValuesSet = difference(usedValuesSet, customOptionsSet); + + if (invalidUsedValuesSet.size === 1) { + errors.push(`Reference in images should be either ${customOptions.join(', ')}. The invalid reference is ${[...invalidUsedValuesSet].join(', ')}`); + } else if (invalidUsedValuesSet.size > 1) { + errors.push(`Reference in images should be either ${customOptions.join(', ')}. The invalid references are ${[...invalidUsedValuesSet].sort().join(', ')}`); + } + return errors; +} + type CustomScreen = Omit; function sanitizeScreens(scenarioPages: TutorialFormType['scenarioPages']) { const screens = scenarioPages.reduce>( @@ -684,34 +682,23 @@ function NewTutorial(props: Props) { ); const handleCocoImport = React.useCallback( - (val) => { - const result = CocoDataset.decode(val); - if (!isRight(result)) { - // eslint-disable-next-line no-console - console.error('Invalid COCO format', result.left); - setError((err) => ({ - ...getErrorObject(err), - [nonFieldError]: 'Invalid COCO format', - })); - return; - } - if (result.right.images.length > MAX_IMAGES) { - setError((err) => ({ - ...getErrorObject(err), - [nonFieldError]: `Too many images ${result.right.images.length} uploaded. Please do not exceed ${MAX_IMAGES} images.`, - })); + (val: CocoDatasetType | undefined) => { + if (isNotDefined(val)) { + setFieldValue( + [], + 'images', + ); return; } - - const newImages = result.right.images.map((image, index) => ({ + const newImages = val.images.map((image, index) => ({ sourceIdentifier: String(image.id), fileName: image.file_name, url: image.flickr_url || image.coco_url, screen: index + 1, - referenceAnswer: 1, + // referenceAnswer: 1, })); setFieldValue( - () => newImages, + newImages, 'images', ); @@ -730,7 +717,7 @@ function NewTutorial(props: Props) { )); setFieldValue(tutorialTaskArray, 'scenarioPages'); }, - [setFieldValue, setError], + [setFieldValue], ); const submissionPending = ( @@ -787,7 +774,13 @@ function NewTutorial(props: Props) { ...subOptions, ].filter(isDefined); - // FIXME: Add warning here for validate image + if (value?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { + return getImagesWarning( + value?.images ?? [], + selectedValues, + ); + } + return getGeoJSONWarning( value?.tutorialTasks, value?.projectType, @@ -795,7 +788,13 @@ function NewTutorial(props: Props) { value?.zoomLevel, ); }, - [value?.tutorialTasks, value?.projectType, value?.customOptions, value?.zoomLevel], + [ + value?.tutorialTasks, + value?.images, + value?.projectType, + value?.customOptions, + value?.zoomLevel, + ], ); const getTileServerUrl = (val: PartialTutorialFormType['tileServer']) => { @@ -821,6 +820,7 @@ function NewTutorial(props: Props) { (newValue: ProjectType | undefined) => { setFieldValue(undefined, 'tutorialTasks'); setFieldValue(undefined, 'scenarioPages'); + setFieldValue(undefined, 'images'); setFieldValue(newValue, 'projectType'); setFieldValue(getDefaultOptions(newValue), 'customOptions'); }, @@ -1061,15 +1061,16 @@ function NewTutorial(props: Props) { - + {(images && images.length > 0) ? (
@@ -1084,6 +1085,7 @@ function NewTutorial(props: Props) { value={image} index={index} onChange={setImageValue} + customOptions={customOptions} error={imagesError?.[image.sourceIdentifier]} disabled={submissionPending || projectTypeEmpty} /> diff --git a/manager-dashboard/app/views/NewTutorial/utils.ts b/manager-dashboard/app/views/NewTutorial/utils.ts index 74fc7711b..e0533080a 100644 --- a/manager-dashboard/app/views/NewTutorial/utils.ts +++ b/manager-dashboard/app/views/NewTutorial/utils.ts @@ -925,11 +925,16 @@ export const tutorialFormSchema: TutorialFormSchema = { ['images'], (formValues) => { // FIXME: Add "unique" constraint for sourceIdentifier and fileName - // FIXME: Add max length constraint if (formValues?.projectType === PROJECT_TYPE_VALIDATE_IMAGE) { return { images: { keySelector: (key) => key.sourceIdentifier, + validation: (values) => { + if (values && values.length > MAX_IMAGES) { + return `Too many images ${values.length}. Please do not exceed ${MAX_IMAGES} images.`; + } + return undefined; + }, member: (): ImageFormSchemaMember => ({ fields: (): ImageSchemaFields => ({ sourceIdentifier: { From d769ac81cae3d1c685c535e0d362dc152305f1c5 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 27 Jun 2025 17:18:38 +0545 Subject: [PATCH 31/35] feat(validate_image): add scripts to generate coco files - generate coco file from images in drive - generate coco file from images in dropbox --- .../user_scripts/generate_coco_from_drive.js | 33 ++++ .../generate_coco_from_dropbox.py | 157 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 manager-dashboard/user_scripts/generate_coco_from_drive.js create mode 100644 manager-dashboard/user_scripts/generate_coco_from_dropbox.py diff --git a/manager-dashboard/user_scripts/generate_coco_from_drive.js b/manager-dashboard/user_scripts/generate_coco_from_drive.js new file mode 100644 index 000000000..957eee989 --- /dev/null +++ b/manager-dashboard/user_scripts/generate_coco_from_drive.js @@ -0,0 +1,33 @@ +function main() { + const exportFileName = 'your_coco_export.json'; + const folderId = 'your_public_folder_id'; + const folder = DriveApp.getFolderById(folderId); + const files = folder.getFiles(); + + const images = []; + + let id = 1; + while (files.hasNext()) { + const file = files.next(); + const name = file.getName(); + const fileId = file.getId(); + // const url = https://drive.google.com/uc?export=view&id=" + fileId; + const url = `https://drive.google.com/thumbnail?id=${fileId}&sz=w1000`; + images.push({ + coco_url: url, + file_name: name, + id, + }); + id += 1; + } + + const exportContent = JSON.stringify({ images }); + const exportFile = DriveApp.createFile( + exportFileName, + exportContent, + MimeType.PLAIN_TEXT, + ); + const exportFileUrl = exportFile.getUrl(); + + Logger.log(`COCO file available at: ${exportFileUrl}`); +} diff --git a/manager-dashboard/user_scripts/generate_coco_from_dropbox.py b/manager-dashboard/user_scripts/generate_coco_from_dropbox.py new file mode 100644 index 000000000..6f3bedfe8 --- /dev/null +++ b/manager-dashboard/user_scripts/generate_coco_from_dropbox.py @@ -0,0 +1,157 @@ +# /// script +# dependencies = [ +# "requests<3", +# ] +# /// +from pathlib import Path +from argparse import ArgumentParser +import requests +import json +import re + +def dropbox_request(endpoint: str, data: object, *, access_token: str): + url = f"https://api.dropboxapi.com/2/{endpoint}" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + res = requests.post( + url, + headers=headers, + data=json.dumps(data), + ) + res.raise_for_status() + return res.json() + +def dropbox_content_request(endpoint: str, path: str, data: object, *, access_token: str): + url = f"https://content.dropboxapi.com/2/{endpoint}" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/octet-stream", + "Dropbox-API-Arg": json.dumps({ + "path": path, + "mode": "overwrite", # overwrite if exists + "autorename": False, + "mute": False + }) + } + res = requests.post( + url, + headers=headers, + data=json.dumps(data).encode("utf-8"), + ) + res.raise_for_status() + return res.json() + +def list_all_files(folder_path: str, *, access_token: str): + ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"} + files = [] + + data = {"path": folder_path, "recursive": False} + response = dropbox_request("files/list_folder", data, access_token=access_token) + + files.extend(response.get("entries", [])) + + while response.get("has_more", False): + cursor = response["cursor"] + response = dropbox_request( + "files/list_folder/continue", + {"cursor": cursor}, + access_token=access_token, + ) + files.extend(response.get("entries", [])) + + # Sort files by name (just in case) + files = sorted(files, key=lambda file: file["name"].lower()) + # Filter out only files (not folders) that are supported + files = [ + file for file in files + if file[".tag"] == "file" and Path(file["name"]).suffix.lower() in ALLOWED_EXTENSIONS + ] + return files + +def share_file_and_get_links(files, *, access_token: str): + total = len(files) + images = [] + for i, file in enumerate(files): + path = file["path_lower"] + actual_path = file["path_display"] + + # First try to list existing shared links + data = {"path": path, "direct_only": True} + print(f"{i + 1}/{total} Getting public URL") + res = dropbox_request( + "sharing/list_shared_links", + data, + access_token=access_token, + ) + if res.get("links"): + link = res["links"][0]["url"] + else: + data = { + "path": path, + "settings": { + "requested_visibility": "public" + } + } + res_create = dropbox_request( + "sharing/create_shared_link_with_settings", + data, + access_token=access_token, + ) + link = res_create["url"] + + raw_url = re.sub(r'&dl=0\b', '', link) + '&raw=1' + + images.append({ + "id": i + 1, + "file_name": actual_path, + "coco_url": raw_url, + }) + return images + + +def main(): + parser = ArgumentParser(description="Generate COCO file from images folder.") + parser.add_argument("access_token", help="Access token for authentication") + parser.add_argument("images_folder", help="Path to the images folder") + parser.add_argument("export_file_name", help="Name of the export COCO file") + + args = parser.parse_args() + + access_token = args.access_token + images_folder = args.images_folder + export_file_name = args.export_file_name + + # Get all the files on given path + files = list_all_files( + images_folder, + access_token=access_token, + ) + + # Share individual file publically and get public link + public_images = share_file_and_get_links( + files, + access_token=access_token, + ) + + # Upload coco format export to dropbox + print("Uploading COCO file") + absolute_export_file_name = str(Path(images_folder) / Path(export_file_name)) + dropbox_content_request( + "files/upload", + absolute_export_file_name, + { "images": public_images }, + access_token=access_token, + ) + + # Get temporary link + res = dropbox_request( + "files/get_temporary_link", + { "path": absolute_export_file_name }, + access_token=access_token, + ) + print(f"COCO file available at {res["link"]}") + +if __name__ == "__main__": + main() From c1c5bb49206adfcbc33a1fa404f75462cbf17058 Mon Sep 17 00:00:00 2001 From: Aditya Khatri Date: Tue, 8 Jul 2025 08:04:35 +0545 Subject: [PATCH 32/35] feat(validate_image): add validate image updates in community dashboard --- .../app/views/StatsBoard/index.tsx | 36 ++++++++++++++----- community-dashboard/docker-compose.yml | 3 +- django/apps/existing_database/models.py | 1 + django/schema.graphql | 1 + 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/community-dashboard/app/views/StatsBoard/index.tsx b/community-dashboard/app/views/StatsBoard/index.tsx index f3b842997..466567094 100644 --- a/community-dashboard/app/views/StatsBoard/index.tsx +++ b/community-dashboard/app/views/StatsBoard/index.tsx @@ -49,6 +49,7 @@ import { ProjectTypeSwipeStatsType, ProjectTypeAreaStatsType, ContributorSwipeStatType, + ProjectTypeEnum, } from '#generated/types'; import { mergeItems } from '#utils/common'; import { @@ -67,17 +68,28 @@ const CHART_BREAKPOINT = 700; export type ActualContributorTimeStatType = ContributorTimeStatType & { totalSwipeTime: number }; const UNKNOWN = '-1'; const BUILD_AREA = 'BUILD_AREA'; +const MEDIA = 'MEDIA'; +const DIGITIZATION = 'DIGITIZATION'; const FOOTPRINT = 'FOOTPRINT'; const CHANGE_DETECTION = 'CHANGE_DETECTION'; +const VALIDATE_IMAGE = 'VALIDATE_IMAGE'; const COMPLETENESS = 'COMPLETENESS'; const STREET = 'STREET'; // FIXME: the name property is not used properly -const projectTypes: Record = { +const projectTypes: Record = { [UNKNOWN]: { color: '#cacaca', name: 'Unknown', }, + [MEDIA]: { + color: '#cacaca', + name: 'Media', + }, + [DIGITIZATION]: { + color: '#cacaca', + name: 'Digitization', + }, [BUILD_AREA]: { color: '#f8a769', name: 'Find', @@ -94,6 +106,10 @@ const projectTypes: Record = { color: '#fb8072', name: 'Completeness', }, + [VALIDATE_IMAGE]: { + color: '#a1b963', + name: 'Validate Image', + }, [STREET]: { color: '#808080', name: 'Street', @@ -376,14 +392,16 @@ function StatsBoard(props: Props) { const sortedProjectSwipeType = useMemo( () => ( swipeByProjectType - ?.map((item) => ({ - ...item, - projectType: ( - isDefined(item.projectType) - && isDefined(projectTypes[item.projectType]) - ) ? item.projectType - : UNKNOWN, - })) + ?.map((item) => { + const projectType: ProjectTypeEnum | '-1' = ( + isDefined(item.projectType) && isDefined(projectTypes[item.projectType]) + ) ? item.projectType : UNKNOWN; + + return ({ + ...item, + projectType, + }); + }) .sort((a, b) => compareNumber(a.totalSwipes, b.totalSwipes, -1)) ?? [] ), [swipeByProjectType], diff --git a/community-dashboard/docker-compose.yml b/community-dashboard/docker-compose.yml index 39ac61dcc..2b548f3bb 100644 --- a/community-dashboard/docker-compose.yml +++ b/community-dashboard/docker-compose.yml @@ -2,7 +2,6 @@ version: '3.3' services: react: - build: . command: sh -c 'yarn install --frozen-lockfile && yarn start' build: context: ./ @@ -15,4 +14,4 @@ services: volumes: - .:/code ports: - - '3080:3080' + - '3081:3081' diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 5bc85e113..319c28b7c 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -69,6 +69,7 @@ class Type(models.IntegerChoices): MEDIA = 5, "Media" DIGITIZATION = 6, "Digitization" STREET = 7, "Street" + VALIDATE_IMAGE = 10, "Validate Image" project_id = models.CharField(primary_key=True, max_length=999) created = models.DateTimeField(blank=True, null=True) diff --git a/django/schema.graphql b/django/schema.graphql index b5596fc46..07b9659c4 100644 --- a/django/schema.graphql +++ b/django/schema.graphql @@ -100,6 +100,7 @@ enum ProjectTypeEnum { MEDIA DIGITIZATION STREET + VALIDATE_IMAGE } type ProjectTypeSwipeStatsType { From 62b3f1a60a1eb58cc1529c56d3ae3860f2fab974 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 11 Jul 2025 18:11:38 +0545 Subject: [PATCH 33/35] feat(aggregates): update aggregates for validate image project - using time_spent_max_allowed value of 6.1 - exluding area calculation --- .../management/commands/update_aggregated_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/apps/aggregated/management/commands/update_aggregated_data.py b/django/apps/aggregated/management/commands/update_aggregated_data.py index dca5896a4..49f536227 100644 --- a/django/apps/aggregated/management/commands/update_aggregated_data.py +++ b/django/apps/aggregated/management/commands/update_aggregated_data.py @@ -55,6 +55,7 @@ WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2 -- FOOTPRINT: Not calculated right now WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1 + WHEN P.project_type = {Project.Type.VALIDATE_IMAGE.value} THEN 6.1 WHEN P.project_type = {Project.Type.STREET.value} THEN 65 ELSE 1 END @@ -111,6 +112,7 @@ WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2 -- FOOTPRINT: Not calculated right now WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1 + WHEN P.project_type = {Project.Type.VALIDATE_IMAGE.value} THEN 6.1 WHEN P.project_type = {Project.Type.STREET.value} THEN 65 ELSE 1 END @@ -136,8 +138,10 @@ G.group_id, ( CASE - -- Hide area for Footprint + -- Hide area for Footprint and Validate Image + -- FIXME: What should we do for Project.Type.STREET.value WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 0 + WHEN P.project_type = {Project.Type.VALIDATE_IMAGE.value} THEN 0 ELSE G.total_area END ) as total_task_group_area, From 7385edfbe3ea652d1e165dd4dc0231fa95e5ab07 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Sun, 13 Jul 2025 13:50:23 +0545 Subject: [PATCH 34/35] chore(ci): Build docker images before-hand --- .github/workflows/actions.yml | 28 ++++++++++++++++------------ django/Dockerfile | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 4609eb697..e3e24b043 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -42,15 +42,28 @@ jobs: exit 1; } + - name: Decrypt Service Account Key File + working-directory: ./ + run: | + openssl enc -aes-256-cbc -d -K "$OPENSSL_KEY" -iv "$OPENSSL_IV" -in ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc -out mapswipe_workers/serviceAccountKey.json + env: + OPENSSL_PASSPHRASE: ${{ secrets.OPENSSL_PASSPHRASE }} + OPENSSL_KEY: ${{ secrets.OPENSSL_KEY }} + OPENSSL_IV: ${{ secrets.OPENSSL_IV }} + + - name: Build docker images + run: | + # Create a mock file for wal-g setup + touch postgres/serviceAccountKey.json + docker compose build postgres firebase_deploy mapswipe_workers_creation django + - name: Setup Postgres Database Container env: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres POSTGRES_DB: postgres run: | - # Create a mock file for wal-g setup - touch postgres/serviceAccountKey.json - docker compose up --build --detach postgres + docker compose up --detach postgres for i in {1..5}; do docker compose exec -T postgres pg_isready && s=0 && break || s=$? && sleep 5; done; (docker compose logs postgres && exit $s) - name: Deploy Firebase Rules and Functions @@ -60,15 +73,6 @@ jobs: run: | docker compose run --rm firebase_deploy sh -c "firebase use $FIREBASE_DB && firebase deploy --token $FIREBASE_TOKEN --only database" - - name: Decrypt Service Account Key File - working-directory: ./ - run: | - openssl enc -aes-256-cbc -d -K "$OPENSSL_KEY" -iv "$OPENSSL_IV" -in ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc -out mapswipe_workers/serviceAccountKey.json - env: - OPENSSL_PASSPHRASE: ${{ secrets.OPENSSL_PASSPHRASE }} - OPENSSL_KEY: ${{ secrets.OPENSSL_KEY }} - OPENSSL_IV: ${{ secrets.OPENSSL_IV }} - - name: Run Tests working-directory: ./mapswipe_workers env: diff --git a/django/Dockerfile b/django/Dockerfile index 4f220f7e6..a6330b600 100644 --- a/django/Dockerfile +++ b/django/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-buster +FROM python:3.10-bullseye LABEL maintainer="Mapswipe info@mapswipe.org" From 1371c5d5b4fd6a1f948d859f251c093c8ebc3121 Mon Sep 17 00:00:00 2001 From: Aditya Khatri Date: Mon, 14 Jul 2025 10:24:19 +0545 Subject: [PATCH 35/35] feat(validate_image): add validate image swipes count in stats group --- .../app/resources/icons/validate-image.svg | 13 +++++++++ .../app/views/StatsBoard/index.tsx | 28 +++++++++++++++++++ .../app/views/StatsBoard/styles.css | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 community-dashboard/app/resources/icons/validate-image.svg diff --git a/community-dashboard/app/resources/icons/validate-image.svg b/community-dashboard/app/resources/icons/validate-image.svg new file mode 100644 index 000000000..7066c5c2b --- /dev/null +++ b/community-dashboard/app/resources/icons/validate-image.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/community-dashboard/app/views/StatsBoard/index.tsx b/community-dashboard/app/views/StatsBoard/index.tsx index 466567094..d04b93212 100644 --- a/community-dashboard/app/views/StatsBoard/index.tsx +++ b/community-dashboard/app/views/StatsBoard/index.tsx @@ -43,6 +43,7 @@ import InformationCard from '#components/InformationCard'; import areaSvg from '#resources/icons/area.svg'; import sceneSvg from '#resources/icons/scene.svg'; import featureSvg from '#resources/icons/feature.svg'; +import validateImageSvg from '#resources/icons/validate-image.svg'; import { ContributorTimeStatType, OrganizationSwipeStatsType, @@ -467,6 +468,10 @@ function StatsBoard(props: Props) { (project) => project.projectType === FOOTPRINT, )?.totalSwipes; + const validateImageTotalSwipes = swipeByProjectType?.find( + (project) => project.projectType === VALIDATE_IMAGE, + )?.totalSwipes; + const organizationColors = scaleOrdinal() .domain(totalSwipesByOrganizationStats?.map( (organization) => (organization.organizationName), @@ -717,6 +722,29 @@ function StatsBoard(props: Props) { subHeading="Compare" variant="stat" /> + + )} + value={( + + )} + label={( +
+ Images Validated +
+ )} + subHeading="Validate Image" + variant="stat" + />
* { flex-basis: 0; flex-grow: 1; - min-width: 12rem; + min-width: 24rem; @media (max-width: 48rem) { min-width: 100%;