diff --git a/src/server/ml_models.js b/src/server/ml_models.js index 19ca7440..90e8761d 100644 --- a/src/server/ml_models.js +++ b/src/server/ml_models.js @@ -68,3 +68,10 @@ read.on('close', () => { // console.log('Session ended.'); process.exit(0); }); + +// function exported for use in server.js +export async function classifyToxicity(input) { + const classifier = await loadToxicityClassifier(); + const output = await classifier(input); + return output; // Format: [ {'toxic', score: 0.99} ] +} diff --git a/src/server/server.js b/src/server/server.js index 5fd426bb..8da4a618 100644 --- a/src/server/server.js +++ b/src/server/server.js @@ -1,3 +1,6 @@ +// importing toxic language ml model +import { classifyToxicity } from './ml_models'; + require('dotenv').config(); const fs = require('fs'); @@ -30,6 +33,12 @@ app.post('/post-question', jsonParser, async (req, res) => { return res.status(400).json({ error: 'Missing required content field' }); } + // checking toxicity + const result = await classifyToxicity(question); + if (result[0].label === 'toxic') { + return res.status(403).json({ error: 'Question rejected due to toxic language' }); + } + // uses time stamp + random string const generateQuestionID = () => `Q-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`; @@ -87,6 +96,12 @@ app.post('/post-answer', jsonParser, async (req, res) => { .json({ error: 'Missing required fields: content or questionID' }); } + // checking toxicity + const result = await classifyToxicity(content); + if (result[0].label === 'toxic') { + return res.status(403).json({ error: 'Answer rejected due to toxic language' }); + } + const generateAnswerID = () => `A-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;