From c1d257e1324262d0275a1cceb1e527f0f0afd567 Mon Sep 17 00:00:00 2001 From: hersveit Date: Wed, 9 Jul 2025 20:39:45 +0300 Subject: [PATCH 1/2] add openobserve logger --- backend/api.env.example | 6 ++ .../apps/api/src/__mocks__/EnvConfigMock.ts | 5 ++ backend/apps/api/src/app.module.ts | 11 ++++ backend/apps/api/src/config/config.model.ts | 15 +++++ backend/apps/api/src/main.ts | 5 +- .../apps/api/src/modules/auth/auth.service.ts | 14 ++++- .../apps/api/src/modules/user/user.service.ts | 9 ++- .../src/interceptor/interceptor.service.ts | 53 ++++++++++++++++- backend/libs/logger/src/index.ts | 2 + backend/libs/logger/src/model.ts | 6 ++ .../src/modules/logger/logger.service.ts | 59 +++++++++++++++++++ .../modules/openobserve/openobserve.model.ts | 20 +++++++ .../modules/openobserve/openobserve.module.ts | 55 +++++++++++++++++ .../openobserve/openobserve.service.ts | 22 +++++++ backend/libs/logger/tsconfig.lib.json | 9 +++ backend/nest-cli.json | 9 +++ backend/package.json | 4 +- backend/pnpm-lock.yaml | 8 +++ backend/tsconfig.json | 4 +- nestjs-starter-kit.code-workspace | 4 ++ 20 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 backend/libs/logger/src/index.ts create mode 100644 backend/libs/logger/src/model.ts create mode 100644 backend/libs/logger/src/modules/logger/logger.service.ts create mode 100644 backend/libs/logger/src/modules/openobserve/openobserve.model.ts create mode 100644 backend/libs/logger/src/modules/openobserve/openobserve.module.ts create mode 100644 backend/libs/logger/src/modules/openobserve/openobserve.service.ts create mode 100644 backend/libs/logger/tsconfig.lib.json diff --git a/backend/api.env.example b/backend/api.env.example index 1287e03..8c57320 100644 --- a/backend/api.env.example +++ b/backend/api.env.example @@ -11,3 +11,9 @@ JWT_SECRET="123" JWT_EXPIRES_IN="2min" JWT_REFRESH_SECRET="321" JWT_REFRESH_EXPIRES_IN="2days" + +OPENOBSERVE_ENDPOINT="http://localhost:5080" +OPENOBSERVE_KEY="" # see key in http://localhost:5080/web/ingestion/custom/logs/curl?org_identifier=default +OPENOBSERVE_USER="admin@email.com" +OPENOBSERVE_ORGANIZATION_ID="nestjs-starter-kit" +OPENOBSERVE_APP_ID="api" diff --git a/backend/apps/api/src/__mocks__/EnvConfigMock.ts b/backend/apps/api/src/__mocks__/EnvConfigMock.ts index d00142a..60e926c 100644 --- a/backend/apps/api/src/__mocks__/EnvConfigMock.ts +++ b/backend/apps/api/src/__mocks__/EnvConfigMock.ts @@ -12,4 +12,9 @@ export const EnvConfigMock: EnvConfig = { JWT_REFRESH_SECRET: 'jwtRefreshSecret', JWT_REFRESH_EXPIRES_IN: '2days', DOMAIN: 'http://localhost:3000', + OPENOBSERVE_ENDPOINT: '', + OPENOBSERVE_KEY: '', + OPENOBSERVE_ORGANIZATION_ID: '', + OPENOBSERVE_APP_ID: '', + OPENOBSERVE_USER: '', }; diff --git a/backend/apps/api/src/app.module.ts b/backend/apps/api/src/app.module.ts index 8f6b3f5..f036fdb 100644 --- a/backend/apps/api/src/app.module.ts +++ b/backend/apps/api/src/app.module.ts @@ -1,4 +1,5 @@ import { ConfigModule, Interceptor } from '@lib/core'; +import { OpenobserveModule } from '@lib/logger'; import { RepositoryModule } from '@lib/repository'; import { Module } from '@nestjs/common'; import { APP_INTERCEPTOR } from '@nestjs/core'; @@ -13,6 +14,16 @@ import { UserModule } from './modules/user/user.module'; inject: [EnvConfig], useFactory: (config: EnvConfig) => config, }), + OpenobserveModule.registerAsync({ + inject: [EnvConfig], + useFactory: (config: EnvConfig) => ({ + endpoint: config.OPENOBSERVE_ENDPOINT, + key: config.OPENOBSERVE_KEY, + organizationId: config.OPENOBSERVE_ORGANIZATION_ID, + appId: config.OPENOBSERVE_APP_ID, + user: config.OPENOBSERVE_USER, + }), + }), UserModule, AuthModule, ], diff --git a/backend/apps/api/src/config/config.model.ts b/backend/apps/api/src/config/config.model.ts index 68259b0..e39937a 100644 --- a/backend/apps/api/src/config/config.model.ts +++ b/backend/apps/api/src/config/config.model.ts @@ -36,4 +36,19 @@ export class EnvConfig { @IsString() DOMAIN: string; + + @IsString() + OPENOBSERVE_ENDPOINT: string; + + @IsString() + OPENOBSERVE_KEY: string; + + @IsString() + OPENOBSERVE_ORGANIZATION_ID: string; + + @IsString() + OPENOBSERVE_APP_ID: string; + + @IsString() + OPENOBSERVE_USER: string; } diff --git a/backend/apps/api/src/main.ts b/backend/apps/api/src/main.ts index 87da526..c4258fb 100644 --- a/backend/apps/api/src/main.ts +++ b/backend/apps/api/src/main.ts @@ -1,3 +1,5 @@ +import { LoggerService } from '@lib/logger'; +import { OpenobserveService } from '@lib/logger/modules/openobserve/openobserve.service'; import { INestApplication } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import cookieParser from 'cookie-parser'; @@ -11,8 +13,9 @@ const enableCorsByEnv = (app: INestApplication) => { }; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create(AppModule, { bufferLogs: true }); app.use(cookieParser()); + app.useLogger(new LoggerService(app.get(OpenobserveService))); enableCorsByEnv(app); const config = app.get(EnvConfig); diff --git a/backend/apps/api/src/modules/auth/auth.service.ts b/backend/apps/api/src/modules/auth/auth.service.ts index fa8695b..74464d3 100644 --- a/backend/apps/api/src/modules/auth/auth.service.ts +++ b/backend/apps/api/src/modules/auth/auth.service.ts @@ -1,7 +1,7 @@ import { UserNotFound, isError } from '@lib/core'; import { RepositoryService } from '@lib/repository'; import { UserEntity } from '@lib/repository/entities/user.entity'; -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; import { SHA256 } from 'crypto-js'; import { v4 } from 'uuid'; @@ -18,6 +18,8 @@ import { GetTokenResult } from './common/auth.model'; @Injectable() export class AuthService { + private readonly logger = new Logger(AuthService.name); + constructor( private readonly jwt: JwtService, private readonly rep: RepositoryService, @@ -96,6 +98,11 @@ export class AuthService { user.refreshTokenHash = refreshTokenHash; await this.rep.user.save(user); + this.logger.log({ + message: 'Confirm email', + payload: { email: user.email, user: user.id, token }, + }); + return { token: accessToken, refreshCookie: @@ -119,6 +126,11 @@ export class AuthService { // `${link}`, // userResult.email, // ); FIXME: Need mail service implementation + + this.logger.log({ + message: 'Send confirm email', + payload: { email: user.email, token, user: user.id }, + }); } private generateToken(id: number) { diff --git a/backend/apps/api/src/modules/user/user.service.ts b/backend/apps/api/src/modules/user/user.service.ts index 5fc969a..269529d 100644 --- a/backend/apps/api/src/modules/user/user.service.ts +++ b/backend/apps/api/src/modules/user/user.service.ts @@ -1,11 +1,13 @@ import { UserNotFound } from '@lib/core'; import { RepositoryService } from '@lib/repository'; -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { SHA256 } from 'crypto-js'; import { UserAlreadyExists } from './common/user.errors'; @Injectable() export class UserService { + private readonly logger = new Logger(UserService.name); + constructor(private readonly rep: RepositoryService) {} async createUser(email: string, password: string) { @@ -13,6 +15,11 @@ export class UserService { return new UserAlreadyExists(email); } + this.logger.log({ + message: 'Create user', + payload: { email }, + }); + const user = this.rep.user.create({ email, hash: SHA256(password).toString(), createdAt: new Date() }); await this.rep.user.save(user); diff --git a/backend/libs/core/src/interceptor/interceptor.service.ts b/backend/libs/core/src/interceptor/interceptor.service.ts index 9524e3c..1f6c990 100644 --- a/backend/libs/core/src/interceptor/interceptor.service.ts +++ b/backend/libs/core/src/interceptor/interceptor.service.ts @@ -22,6 +22,16 @@ export class Interceptor implements NestInterceptor { async intercept(context: ExecutionContext, next: CallHandler): Promise> { const res: Response = context.switchToHttp().getResponse(); + const req = context.switchToHttp().getRequest(); + + this.logger.log({ + message: 'Request', + payload: { + method: req.method, + url: req.url, + user: req.user, + }, + }); return next .handle() @@ -39,6 +49,18 @@ export class Interceptor implements NestInterceptor { if (data.body.success) { res.status(200); + + this.logger.log({ + message: 'Response', + payload: { + status: data.status, + method: req.method, + url: req.url, + body: data.body, + user: req.user, + }, + }); + return data.body; } @@ -60,19 +82,44 @@ export class Interceptor implements NestInterceptor { status = err.getStatus(); } - this.logger.error(`INTERNAL_SERVER_ERROR Error(): ${message};\nStack: ${err.stack}`); + this.logger.error({ + message: 'INTERNAL_SERVER_ERROR', + payload: { + message, + stack: err.stack, + method: req.method, + url: req.url, + user: req.user, + }, + }); res.status(status); return of(message); } if (err?.body && !err.body.success) { - this.logger.error(`Error: ${JSON.stringify(err)}`); + this.logger.error({ + message: 'Error', + payload: { + body: err.body, + method: req.method, + url: req.url, + user: req.user, + }, + }); res.status(err.status); return of(objToString(err.body)); } const unknownError = objToString(err); - this.logger.error(`INTERNAL_SERVER_ERROR unknown: ${unknownError}`); + this.logger.error({ + message: 'UNKNOWN INTERNAL_SERVER_ERROR', + payload: { + error: unknownError, + method: req.method, + url: req.url, + user: req.user, + }, + }); res.status(HttpStatus.INTERNAL_SERVER_ERROR); return of(unknownError); }), diff --git a/backend/libs/logger/src/index.ts b/backend/libs/logger/src/index.ts new file mode 100644 index 0000000..f4b0c7a --- /dev/null +++ b/backend/libs/logger/src/index.ts @@ -0,0 +1,2 @@ +export { LoggerService } from './modules/logger/logger.service'; +export { OpenobserveModule } from './modules/openobserve/openobserve.module'; diff --git a/backend/libs/logger/src/model.ts b/backend/libs/logger/src/model.ts new file mode 100644 index 0000000..9f681e0 --- /dev/null +++ b/backend/libs/logger/src/model.ts @@ -0,0 +1,6 @@ +export type Log = { + level: 'log' | 'error' | 'warn' | 'http_exception'; + payload: any; + serviceName: string; + time: Date; +}; diff --git a/backend/libs/logger/src/modules/logger/logger.service.ts b/backend/libs/logger/src/modules/logger/logger.service.ts new file mode 100644 index 0000000..5ddb797 --- /dev/null +++ b/backend/libs/logger/src/modules/logger/logger.service.ts @@ -0,0 +1,59 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { Injectable, LoggerService as NestLoggerService } from '@nestjs/common'; +import { format } from 'date-fns'; +import { OpenobserveService } from '../openobserve/openobserve.service'; + +const red = '\x1b[31m'; +const yellow = '\x1b[33m'; +const green = '\x1b[32m'; +const reset = '\x1b[0m'; + +@Injectable() +export class LoggerService implements NestLoggerService { + constructor(private readonly openobserve: OpenobserveService) {} + + log(message: any, serviceName1?: string, serviceName2?: string) { + this.logToOpenobserve('log', message, serviceName1, serviceName2); + } + + error(message: any, serviceName1?: string, serviceName2?: string) { + this.logToOpenobserve('error', message, serviceName1, serviceName2); + } + + warn(message: any, serviceName1?: string, serviceName2?: string) { + this.logToOpenobserve('warn', message, serviceName1, serviceName2); + } + + private logToOpenobserve( + level: 'log' | 'error' | 'warn', + message: any, + serviceName1: string | undefined, + serviceName2: string | undefined, + ) { + const serviceName = serviceName1 || serviceName2 || 'unknown'; + const time = new Date(); + this.openobserve + .log({ + level, + payload: { message }, + serviceName, + time, + }) + .finally(() => this.coloredLog(level, serviceName, message, time)); + } + + private coloredLog(level: 'log' | 'error' | 'warn', serviceName: string, message: any, time: Date) { + const payload = typeof message === 'object' ? JSON.stringify(message) : String(message); + + // eslint-disable-next-line no-console + console.log( + format(time, 'yyyy-MM-dd, HH:mm:ss.SSS'), + green, + `[${serviceName}]`, + level === 'error' ? red : level === 'warn' ? yellow : reset, + payload, + reset, + ); + } +} diff --git a/backend/libs/logger/src/modules/openobserve/openobserve.model.ts b/backend/libs/logger/src/modules/openobserve/openobserve.model.ts new file mode 100644 index 0000000..62465e8 --- /dev/null +++ b/backend/libs/logger/src/modules/openobserve/openobserve.model.ts @@ -0,0 +1,20 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { ModuleMetadata, Provider, Type } from '@nestjs/common'; + +export interface OpenobserveModuleOptions { + endpoint: string; + key: string; + organizationId: string; + appId: string; + user: string; +} + +export interface OpenobserveModuleAsyncOptions extends Pick { + name?: string; + useExisting?: Type; + useClass?: Type; + useFactory?: (...args: Array) => Promise | OpenobserveModuleOptions; + inject?: Array; + extraProviders?: Array; +} diff --git a/backend/libs/logger/src/modules/openobserve/openobserve.module.ts b/backend/libs/logger/src/modules/openobserve/openobserve.module.ts new file mode 100644 index 0000000..36a24a1 --- /dev/null +++ b/backend/libs/logger/src/modules/openobserve/openobserve.module.ts @@ -0,0 +1,55 @@ +import { DynamicModule, Module, Provider } from '@nestjs/common'; +import { OpenobserveModuleAsyncOptions, OpenobserveModuleOptions } from './openobserve.model'; +import { OpenobserveService } from './openobserve.service'; + +@Module({}) +export class OpenobserveModule { + public static registerAsync(options: OpenobserveModuleAsyncOptions): DynamicModule { + const asyncProviders = this.createAsyncProviders(options); + + return { + module: OpenobserveModule, + imports: options.imports || [], + providers: [...asyncProviders, OpenobserveService, ...(options.extraProviders || [])], + exports: [OpenobserveService], + }; + } + + private static createAsyncProviders(options: OpenobserveModuleAsyncOptions): Array { + if (options.useFactory) { + return [ + { + provide: 'OPENOBSERVE_CONFIG', + useFactory: options.useFactory, + inject: options.inject || [], + }, + ]; + } + + if (options.useClass) { + return [ + { + provide: 'OPENOBSERVE_CONFIG', + useFactory: async (optionsFactory: OpenobserveModuleOptions) => optionsFactory, + inject: [options.useClass], + }, + { + provide: options.useClass, + useClass: options.useClass, + }, + ]; + } + + if (options.useExisting) { + return [ + { + provide: 'OPENOBSERVE_CONFIG', + useFactory: async (optionsFactory: OpenobserveModuleOptions) => optionsFactory, + inject: [options.useExisting], + }, + ]; + } + + throw new Error('Invalid async options'); + } +} diff --git a/backend/libs/logger/src/modules/openobserve/openobserve.service.ts b/backend/libs/logger/src/modules/openobserve/openobserve.service.ts new file mode 100644 index 0000000..33d7252 --- /dev/null +++ b/backend/libs/logger/src/modules/openobserve/openobserve.service.ts @@ -0,0 +1,22 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Log } from '../../model'; +import { OpenobserveModuleOptions } from './openobserve.model'; + +@Injectable() +export class OpenobserveService { + constructor(@Inject('OPENOBSERVE_CONFIG') private readonly config: OpenobserveModuleOptions) {} + + public log(log: Log) { + return fetch(`${this.config.endpoint}/api/${this.config.organizationId}/${this.config.appId}/_json`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${btoa(`${this.config.user}:${this.config.key}`)}`, + }, + body: JSON.stringify({ + ...log, + container: process.env.HOSTNAME, + }), + }); + } +} diff --git a/backend/libs/logger/tsconfig.lib.json b/backend/libs/logger/tsconfig.lib.json new file mode 100644 index 0000000..7982fb9 --- /dev/null +++ b/backend/libs/logger/tsconfig.lib.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "declaration": true, + "outDir": "../../dist/libs/logger" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] +} diff --git a/backend/nest-cli.json b/backend/nest-cli.json index 58cdd8e..78e54d3 100644 --- a/backend/nest-cli.json +++ b/backend/nest-cli.json @@ -43,6 +43,15 @@ "compilerOptions": { "tsConfigPath": "apps/admin/tsconfig.app.json" } + }, + "logger": { + "type": "library", + "root": "libs/logger", + "entryFile": "index", + "sourceRoot": "libs/logger/src", + "compilerOptions": { + "tsConfigPath": "libs/logger/tsconfig.lib.json" + } } }, "monorepo": true, diff --git a/backend/package.json b/backend/package.json index 06c889c..33a4c45 100644 --- a/backend/package.json +++ b/backend/package.json @@ -33,6 +33,7 @@ "class-validator": "^0.14.1", "cookie-parser": "^1.4.6", "crypto-js": "^4.2.0", + "date-fns": "^4.1.0", "dotenv": "^16.4.5", "nest-typed-config": "^2.9.3", "nestjs-cls": "^4.3.0", @@ -95,7 +96,8 @@ ], "moduleNameMapper": { "^@lib/repository(|/.*)$": "/libs/repository/src/$1", - "^@lib/core(|/.*)$": "/libs/core/src/$1" + "^@lib/core(|/.*)$": "/libs/core/src/$1", + "^@lib/logger(|/.*)$": "/libs/logger/src/$1" } } } diff --git a/backend/pnpm-lock.yaml b/backend/pnpm-lock.yaml index 91eb970..ffca117 100644 --- a/backend/pnpm-lock.yaml +++ b/backend/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: crypto-js: specifier: ^4.2.0 version: 4.2.0 + date-fns: + specifier: ^4.1.0 + version: 4.1.0 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -1306,6 +1309,9 @@ packages: crypto-js@4.2.0: resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + date-fns@4.1.0: + resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + dayjs@1.11.11: resolution: {integrity: sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==} @@ -4739,6 +4745,8 @@ snapshots: crypto-js@4.2.0: {} + date-fns@4.1.0: {} + dayjs@1.11.11: {} debug@2.6.9: diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 9ca7934..918a261 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -22,7 +22,9 @@ "@lib/repository": ["./libs/repository/src"], "@lib/repository/*": ["./libs/repository/src/*"], "@lib/core": ["./libs/core/src"], - "@lib/core/*": ["./libs/core/src/*"] + "@lib/core/*": ["./libs/core/src/*"], + "@lib/logger": ["./libs/logger/src"], + "@lib/logger/*": ["./libs/logger/src/*"] } } } diff --git a/nestjs-starter-kit.code-workspace b/nestjs-starter-kit.code-workspace index fe10217..8354496 100644 --- a/nestjs-starter-kit.code-workspace +++ b/nestjs-starter-kit.code-workspace @@ -23,6 +23,10 @@ { "path": "backend/libs/repository", "name": "lib/repository" + }, + { + "path": "backend/libs/logger", + "name": "lib/logger" } ], "settings": { From 0ca0afe4e2872f5342d8d1a86b348315a2e47500 Mon Sep 17 00:00:00 2001 From: hersveit Date: Wed, 9 Jul 2025 20:39:57 +0300 Subject: [PATCH 2/2] fix repository --- backend/libs/repository/src/repository.module.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/libs/repository/src/repository.module.ts b/backend/libs/repository/src/repository.module.ts index d941618..23ff006 100644 --- a/backend/libs/repository/src/repository.module.ts +++ b/backend/libs/repository/src/repository.module.ts @@ -6,6 +6,7 @@ import { TransactionalAdapterTypeOrm } from '@nestjs-cls/transactional-adapter-t import { DynamicModule, Global, Module } from '@nestjs/common'; import { TypeOrmModule, getDataSourceToken } from '@nestjs/typeorm'; import { ClsModule } from 'nestjs-cls'; +import { RepositoryService } from './repository.service'; import { RepositoryModuleOptions } from './types/module.types'; export const entities = [UserEntity, BalanceEntity, ItemEntity, AdminEntity]; @@ -49,6 +50,8 @@ export class RepositoryModule { ], }), ], + providers: [RepositoryService], + exports: [RepositoryService], }; } }