Skip to content

Commit 28cded3

Browse files
alaa-eddine-kalaa-eddine-k
authored
Merge pull request #121 from SmythOS/dev
better logging for LLM connectors
2 parents 75b6b23 + 34b7909 commit 28cded3

File tree

12 files changed

+295
-201
lines changed

12 files changed

+295
-201
lines changed

‎packages/core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@smythos/sre",
3-
"version": "1.5.61",
3+
"version": "1.5.63",
44
"description": "Smyth Runtime Environment",
55
"author": "Alaa-eddine KADDOURI",
66
"license": "MIT",

‎packages/core/src/subsystems/LLMManager/LLM.service/LLMConnector.ts‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,32 @@
11
import { Connector } from '@sre/Core/Connector.class';
22
import { ConnectorService } from '@sre/Core/ConnectorsService';
3+
import { JSONContent } from '@sre/helpers/JsonContent.helper';
34
import { Logger } from '@sre/helpers/Log.helper';
5+
import { ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
46
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
5-
import { JSONContent } from '@sre/helpers/JsonContent.helper';
7+
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
8+
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
69
import type {
7-
TLLMConnectorParams,
8-
TLLMMessageBlock,
9-
TLLMToolResultMessageBlock,
10-
ToolData,
1110
APIKeySource,
12-
TLLMModel,
1311
ILLMRequestFuncParams,
1412
TLLMChatResponse,
13+
TLLMConnectorParams,
14+
TLLMMessageBlock,
15+
TLLMModel,
16+
TLLMPreparedParams,
1517
TLLMRequestBody,
18+
TLLMToolResultMessageBlock,
19+
ToolData,
1620
TOpenAIToolsInfo,
17-
TxAIToolsInfo,
18-
TLLMPreparedParams,
1921
TToolsInfo,
22+
TxAIToolsInfo,
2023
} from '@sre/types/LLM.types';
24+
import { TCustomLLMModel } from '@sre/types/LLM.types';
2125
import EventEmitter from 'events';
2226
import { Readable } from 'stream';
23-
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
24-
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
25-
import { TCustomLLMModel } from '@sre/types/LLM.types';
26-
import config from '@sre/config';
27-
import { ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
2827
import { getLLMCredentials } from './LLMCredentials.helper';
2928

30-
const console = Logger('LLMConnector');
29+
const logger = Logger('LLMConnector');
3130

3231
export interface ILLMConnectorRequest {
3332
// chatRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any>;
@@ -100,11 +99,12 @@ export abstract class LLMConnector extends Connector {
10099
this.vaultConnector = ConnectorService.getVaultConnector();
101100

102101
if (!this.vaultConnector || !this.vaultConnector.valid) {
103-
console.warn(`Vault Connector unavailable for ${candidate.id} `);
102+
logger.warn(`Vault Connector unavailable for ${candidate.id} `);
104103
}
105104

106105
const _request: ILLMConnectorRequest = {
107106
request: async (params: TLLMConnectorParams) => {
107+
logger.debug(`request ${this.name}`, candidate);
108108
const preparedParams = await this.prepareParams(candidate, params);
109109

110110
const provider = preparedParams.modelInfo.provider;
@@ -129,6 +129,7 @@ export abstract class LLMConnector extends Connector {
129129
return response;
130130
},
131131
streamRequest: async (params: TLLMConnectorParams) => {
132+
logger.debug(`streamRequest ${this.name}`, candidate);
132133
const preparedParams = await this.prepareParams(candidate, params);
133134

134135
const provider = preparedParams.modelInfo.provider?.toLowerCase();
@@ -156,6 +157,7 @@ export abstract class LLMConnector extends Connector {
156157
},
157158

158159
imageGenRequest: async (params: any) => {
160+
logger.debug(`imageGenRequest ${this.name}`, candidate);
159161
const preparedParams = await this.prepareParams(candidate, params);
160162

161163
const response = await this.imageGenRequest({
@@ -175,6 +177,7 @@ export abstract class LLMConnector extends Connector {
175177
return response;
176178
},
177179
imageEditRequest: async (params: any) => {
180+
logger.debug(`imageEditRequest ${this.name}`, candidate);
178181
const preparedParams = await this.prepareParams(candidate, params);
179182

180183
const response = await this.imageEditRequest({

‎packages/core/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ import { JSONContent } from '@sre/helpers/JsonContent.helper';
2525
import { LLMConnector } from '../LLMConnector';
2626
import { SystemEvents } from '@sre/Core/SystemEvents';
2727
import { SUPPORTED_MIME_TYPES_MAP } from '@sre/constants';
28+
import { Logger } from '@sre/helpers/Log.helper';
29+
30+
const logger = Logger('AnthropicConnector');
2831

2932
const PREFILL_TEXT_FOR_JSON_RESPONSE = '{';
3033
const LEGACY_THINKING_MODELS = ['smythos/claude-3.7-sonnet-thinking', 'claude-3.7-sonnet-thinking'];
@@ -49,6 +52,7 @@ export class AnthropicConnector extends LLMConnector {
4952

5053
protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
5154
try {
55+
logger.debug(`request ${this.name}`, acRequest.candidate);
5256
const anthropic = await this.getClient(context);
5357
const result = await anthropic.messages.create(body);
5458
const message: Anthropic.MessageParam = {
@@ -104,12 +108,14 @@ export class AnthropicConnector extends LLMConnector {
104108
usage,
105109
};
106110
} catch (error) {
111+
logger.error(`request ${this.name}`, error, acRequest.candidate);
107112
throw error;
108113
}
109114
}
110115

111116
protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
112117
try {
118+
logger.debug(`streamRequest ${this.name}`, acRequest.candidate);
113119
const emitter = new EventEmitter();
114120
const usage_data = [];
115121

@@ -199,6 +205,7 @@ export class AnthropicConnector extends LLMConnector {
199205

200206
return emitter;
201207
} catch (error: any) {
208+
logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
202209
throw error;
203210
}
204211
}

‎packages/core/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ import { isJSONString } from '@sre/utils/general.utils';
3030
import { LLMConnector } from '../LLMConnector';
3131
import { JSONContent } from '@sre/helpers/JsonContent.helper';
3232
import { SystemEvents } from '@sre/Core/SystemEvents';
33+
import { Logger } from '@sre/helpers/Log.helper';
34+
35+
const logger = Logger('BedrockConnector');
3336

3437
// TODO [Forhad]: Need to adjust some type definitions
3538

@@ -50,6 +53,7 @@ export class BedrockConnector extends LLMConnector {
5053

5154
protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
5255
try {
56+
logger.debug(`request ${this.name}`, acRequest.candidate);
5357
const bedrock = await this.getClient(context);
5458
const command = new ConverseCommand(body);
5559
const response: ConverseCommandOutput = await bedrock.send(command);
@@ -91,6 +95,7 @@ export class BedrockConnector extends LLMConnector {
9195
usage,
9296
};
9397
} catch (error: any) {
98+
logger.error(`request ${this.name}`, error, acRequest.candidate);
9499
throw error?.error || error;
95100
}
96101
}
@@ -99,6 +104,7 @@ export class BedrockConnector extends LLMConnector {
99104
const emitter = new EventEmitter();
100105

101106
try {
107+
logger.debug(`streamRequest ${this.name}`, acRequest.candidate);
102108
const bedrock = await this.getClient(context);
103109
const command = new ConverseStreamCommand(body);
104110
const response: ConverseStreamCommandOutput = await bedrock.send(command);
@@ -189,6 +195,7 @@ export class BedrockConnector extends LLMConnector {
189195
return emitter;
190196
} catch (error: unknown) {
191197
const typedError = error as Error;
198+
logger.error(`streamRequest ${this.name}`, typedError, acRequest.candidate);
192199
emitter.emit(TLLMEvent.Error, typedError?.['error'] || typedError);
193200
return emitter;
194201
}

‎packages/core/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts‎

Lines changed: 49 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2,54 +2,69 @@ import { JSONContent } from '@sre/helpers/JsonContent.helper';
22
import { LLMConnector } from '../LLMConnector';
33
import EventEmitter from 'events';
44
import { APIKeySource, ILLMRequestFuncParams, TLLMChatResponse, TLLMPreparedParams } from '@sre/types/LLM.types';
5+
import { Logger } from '@sre/helpers/Log.helper';
6+
7+
const logger = Logger('EchoConnector');
58

69
export class EchoConnector extends LLMConnector {
710
public name = 'LLM:Echo';
811

912
protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
10-
const content = body?.messages?.[0]?.content; // As Echo model only used in PromptGenerator so we can assume the first message is the user message to echo
11-
return {
12-
content,
13-
finishReason: 'stop',
14-
useTool: false,
15-
toolsData: [],
16-
message: { content, role: 'assistant' },
17-
usage: {},
18-
} as TLLMChatResponse;
13+
try {
14+
logger.debug(`request ${this.name}`, acRequest.candidate);
15+
const content = body?.messages?.[0]?.content; // As Echo model only used in PromptGenerator so we can assume the first message is the user message to echo
16+
return {
17+
content,
18+
finishReason: 'stop',
19+
useTool: false,
20+
toolsData: [],
21+
message: { content, role: 'assistant' },
22+
usage: {},
23+
} as TLLMChatResponse;
24+
} catch (error) {
25+
logger.error(`request ${this.name}`, error, acRequest.candidate);
26+
throw error;
27+
}
1928
}
2029

2130
protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
22-
const emitter = new EventEmitter();
23-
let content = '';
31+
try {
32+
logger.debug(`streamRequest ${this.name}`, acRequest.candidate);
33+
const emitter = new EventEmitter();
34+
let content = '';
2435

25-
if (Array.isArray(body?.messages)) {
26-
content = body?.messages?.filter((m) => m.role === 'user').pop()?.content;
27-
}
28-
//params?.messages?.[0]?.content;
36+
if (Array.isArray(body?.messages)) {
37+
content = body?.messages?.filter((m) => m.role === 'user').pop()?.content;
38+
}
39+
//params?.messages?.[0]?.content;
2940

30-
// Process stream asynchronously as we need to return emitter immediately
31-
(async () => {
32-
// Simulate streaming by splitting content into chunks
33-
const chunks = content.split(' ');
41+
// Process stream asynchronously as we need to return emitter immediately
42+
(async () => {
43+
// Simulate streaming by splitting content into chunks
44+
const chunks = content.split(' ');
3445

35-
for (let i = 0; i < chunks.length; i++) {
36-
// Simulate network delay
37-
await new Promise((resolve) => setTimeout(resolve, 50));
46+
for (let i = 0; i < chunks.length; i++) {
47+
// Simulate network delay
48+
await new Promise((resolve) => setTimeout(resolve, 50));
3849

39-
const isLastChunk = i === chunks.length - 1;
40-
// Add space between chunks except for the last one to avoid trailing space in file URLs
41-
const delta = { content: chunks[i] + (isLastChunk ? '' : ' ') };
42-
emitter.emit('data', delta);
43-
emitter.emit('content', delta.content);
44-
}
50+
const isLastChunk = i === chunks.length - 1;
51+
// Add space between chunks except for the last one to avoid trailing space in file URLs
52+
const delta = { content: chunks[i] + (isLastChunk ? '' : ' ') };
53+
emitter.emit('data', delta);
54+
emitter.emit('content', delta.content);
55+
}
4556

46-
// Emit end event after all chunks are processed
47-
setTimeout(() => {
48-
emitter.emit('end', [], []); // Empty arrays for toolsData and usage_data
49-
}, 100);
50-
})();
57+
// Emit end event after all chunks are processed
58+
setTimeout(() => {
59+
emitter.emit('end', [], []); // Empty arrays for toolsData and usage_data
60+
}, 100);
61+
})();
5162

52-
return emitter;
63+
return emitter;
64+
} catch (error) {
65+
logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
66+
throw error;
67+
}
5368
}
5469

5570
protected async reqBodyAdapter(params: TLLMPreparedParams): Promise<any> {

‎packages/core/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,12 @@ import { LLMHelper } from '@sre/LLMManager/LLM.helper';
3232

3333
import { SystemEvents } from '@sre/Core/SystemEvents';
3434
import { SUPPORTED_MIME_TYPES_MAP } from '@sre/constants';
35+
import { Logger } from '@sre/helpers/Log.helper';
3536

3637
import { LLMConnector } from '../LLMConnector';
3738

39+
const logger = Logger('GoogleAIConnector');
40+
3841
const MODELS_SUPPORT_SYSTEM_INSTRUCTION = [
3942
'gemini-1.5-pro-exp-0801',
4043
'gemini-1.5-pro-latest',
@@ -76,6 +79,7 @@ export class GoogleAIConnector extends LLMConnector {
7679

7780
protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
7881
try {
82+
logger.debug(`request ${this.name}`, acRequest.candidate);
7983
const prompt = body.messages;
8084
delete body.messages;
8185

@@ -121,11 +125,13 @@ export class GoogleAIConnector extends LLMConnector {
121125
usage,
122126
};
123127
} catch (error: any) {
128+
logger.error(`request ${this.name}`, error, acRequest.candidate);
124129
throw error;
125130
}
126131
}
127132

128133
protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
134+
logger.debug(`streamRequest ${this.name}`, acRequest.candidate);
129135
const emitter = new EventEmitter();
130136

131137
const prompt = body.messages;
@@ -189,6 +195,7 @@ export class GoogleAIConnector extends LLMConnector {
189195

190196
return emitter;
191197
} catch (error: any) {
198+
logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
192199
throw error;
193200
}
194201
}

0 commit comments

Comments
 (0)