Merge branch 'develop' into kinesis

* develop:
  docs: add S3 as a supported model
  docs: add CW Logs as a supported envelope
  fix: cloudwatch logs envelope typo
  docs: add CW Logs as a supported model
  docs: add Alb as a supported model
  docs: shadow sidebar to remain expanded
  cr fixes
  feat: Add cloudwatch lambda event support to Parser utility
  feat: Add alb lambda event support to Parser utility #228
  feat: Add S3 lambda event support to Parser utility #224

Author: heitorlessa

aws_lambda_powertools/utilities/parser/envelopes/__init__.py

@@ -1,11 +1,13 @@
 from .base import BaseEnvelope
+from .cloudwatch import CloudWatchLogsEnvelope
 from .dynamodb import DynamoDBStreamEnvelope
 from .event_bridge import EventBridgeEnvelope
 from .kinesis import KinesisDataStreamEnvelope
 from .sns import SnsEnvelope
 from .sqs import SqsEnvelope
 
 __all__ = [
+    "CloudWatchLogsEnvelope",
     "DynamoDBStreamEnvelope",
     "EventBridgeEnvelope",
     "KinesisDataStreamEnvelope",

aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from ..models import CloudWatchLogsModel
+from ..types import Model
+from .base import BaseEnvelope
+
+logger = logging.getLogger(__name__)
+
+
+class CloudWatchLogsEnvelope(BaseEnvelope):
+    """CloudWatch Envelope to extract a List of log records.
+
+    The record's body parameter is a string (after being base64 decoded and gzipped),
+    though it can also be a JSON encoded string.
+    Regardless of its type it'll be parsed into a BaseModel object.
+
+    Note: The record will be parsed the same way so if model is str
+    """
+
+    def parse(self, data: Optional[Union[Dict[str, Any], Any]], model: Model) -> List[Optional[Model]]:
+        """Parses records found with model provided
+
+        Parameters
+        ----------
+        data : Dict
+            Lambda event to be parsed
+        model : Model
+            Data model provided to parse after extracting data using envelope
+
+        Returns
+        -------
+        List
+            List of records parsed with model provided
+        """
+        logger.debug(f"Parsing incoming data with SNS model {CloudWatchLogsModel}")
+        parsed_envelope = CloudWatchLogsModel.parse_obj(data)
+        logger.debug(f"Parsing CloudWatch records in `body` with {model}")
+        output = []
+        for record in parsed_envelope.awslogs.decoded_data.logEvents:
+            output.append(self._parse(data=record.message, model=model))
+        return output

aws_lambda_powertools/utilities/parser/models/__init__.py

@@ -1,3 +1,5 @@
+from .alb import AlbModel, AlbRequestContext, AlbRequestContextData
+from .cloudwatch import CloudWatchLogsData, CloudWatchLogsDecode, CloudWatchLogsLogEvent, CloudWatchLogsModel
 from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel
 from .event_bridge import EventBridgeModel
 from .kinesis import KinesisDataStreamModel, KinesisDataStreamRecord, KinesisDataStreamRecordPayload
@@ -6,13 +8,22 @@
 from .sqs import SqsModel, SqsRecordModel
 
 __all__ = [
+    "CloudWatchLogsData",
+    "CloudWatchLogsDecode",
+    "CloudWatchLogsLogEvent",
+    "CloudWatchLogsModel",
+    "AlbModel",
+    "AlbRequestContext",
+    "AlbRequestContextData",
     "DynamoDBStreamModel",
     "EventBridgeModel",
     "DynamoDBStreamChangedRecordModel",
     "DynamoDBStreamRecordModel",
     "KinesisDataStreamModel",
     "KinesisDataStreamRecord",
     "KinesisDataStreamRecordPayload",
+    "S3Model",
+    "S3RecordModel",
     "SesModel",
     "SesRecordModel",
     "SnsModel",

aws_lambda_powertools/utilities/parser/models/alb.py

@@ -0,0 +1,21 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class AlbRequestContextData(BaseModel):
+    targetGroupArn: str
+
+
+class AlbRequestContext(BaseModel):
+    elb: AlbRequestContextData
+
+
+class AlbModel(BaseModel):
+    httpMethod: str
+    path: str
+    body: str
+    isBase64Encoded: bool
+    headers: Dict[str, str]
+    queryStringParameters: Dict[str, str]
+    requestContext: AlbRequestContext

aws_lambda_powertools/utilities/parser/models/cloudwatch.py

@@ -0,0 +1,44 @@
+import base64
+import json
+import logging
+import zlib
+from datetime import datetime
+from typing import List
+
+from pydantic import BaseModel, Field, validator
+
+logger = logging.getLogger(__name__)
+
+
+class CloudWatchLogsLogEvent(BaseModel):
+    id: str  # noqa AA03 VNE003
+    timestamp: datetime
+    message: str
+
+
+class CloudWatchLogsDecode(BaseModel):
+    messageType: str
+    owner: str
+    logGroup: str
+    logStream: str
+    subscriptionFilters: List[str]
+    logEvents: List[CloudWatchLogsLogEvent]
+
+
+class CloudWatchLogsData(BaseModel):
+    decoded_data: CloudWatchLogsDecode = Field(None, alias="data")
+
+    @validator("decoded_data", pre=True)
+    def prepare_data(cls, value):
+        try:
+            logger.debug("Decoding base64 cloudwatch log data before parsing")
+            payload = base64.b64decode(value)
+            logger.debug("Decompressing cloudwatch log data before parsing")
+            uncompressed = zlib.decompress(payload, zlib.MAX_WBITS | 32)
+            return json.loads(uncompressed.decode("utf-8"))
+        except Exception:
+            raise ValueError("unable to decompress data")
+
+
+class CloudWatchLogsModel(BaseModel):
+    awslogs: CloudWatchLogsData

aws_lambda_powertools/utilities/parser/models/s3.py

@@ -0,0 +1,72 @@
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel
+from pydantic.fields import Field
+from pydantic.networks import IPvAnyNetwork
+from pydantic.types import PositiveInt
+from typing_extensions import Literal
+
+
+class S3EventRecordGlacierRestoreEventData(BaseModel):
+    lifecycleRestorationExpiryTime: datetime
+    lifecycleRestoreStorageClass: str
+
+
+class S3EventRecordGlacierEventData(BaseModel):
+    restoreEventData: S3EventRecordGlacierRestoreEventData
+
+
+class S3Identity(BaseModel):
+    principalId: str
+
+
+class S3RequestParameters(BaseModel):
+    sourceIPAddress: IPvAnyNetwork
+
+
+class S3ResponseElements(BaseModel):
+    x_amz_request_id: str = Field(None, alias="x-amz-request-id")
+    x_amz_id_2: str = Field(None, alias="x-amz-id-2")
+
+
+class S3OwnerIdentify(BaseModel):
+    principalId: str
+
+
+class S3Bucket(BaseModel):
+    name: str
+    ownerIdentity: S3OwnerIdentify
+    arn: str
+
+
+class S3Object(BaseModel):
+    key: str
+    size: PositiveInt
+    eTag: str
+    sequencer: str
+    versionId: Optional[str]
+
+
+class S3Message(BaseModel):
+    s3SchemaVersion: str
+    configurationId: str
+    bucket: S3Bucket
+    object: S3Object  # noqa: A003,VNE003
+
+
+class S3RecordModel(BaseModel):
+    eventVersion: str
+    eventSource: Literal["aws:s3"]
+    awsRegion: str
+    eventTime: datetime
+    eventName: str
+    userIdentity: S3Identity
+    requestParameters: S3RequestParameters
+    responseElements: S3ResponseElements
+    s3: S3Message
+    glacierEventData: Optional[S3EventRecordGlacierEventData]
+
+
+class S3Model(BaseModel):
+    Records: List[S3RecordModel]

docs/content/utilities/parser.mdx

@@ -156,6 +156,9 @@ Model name | Description
 **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams
 **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge
 **SqsModel** | Lambda Event Source payload for Amazon SQS
+**AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer
+**CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs
+**S3Model** | Lambda Event Source payload for Amazon S3
 
 You can extend them to include your own models, and yet have all other known fields parsed along the way.
 
@@ -292,6 +295,7 @@ Envelope name | Behaviour | Return
 **DynamoDBStreamEnvelope** | 1. Parses data using `DynamoDBStreamModel`. <br/> 2. Parses records in `NewImage` and `OldImage` keys using your model. <br/> 3. Returns a list with a dictionary containing `NewImage` and `OldImage` keys | `List[Dict[str, Optional[Model]]]`
 **EventBridgeEnvelope** | 1. Parses data using `EventBridgeModel`. <br/> 2. Parses `detail` key using your model and returns it. | `Model`
 **SqsEnvelope** | 1. Parses data using `SqsModel`. <br/> 2. Parses records in `body` key using your model and return them in a list. | `List[Model]`
+**CloudWatchLogsEnvelope** | 1. Parses data using `CloudwatchLogsModel` which will base64 decode and decompress it. <br/> 2. Parses records in `message` key using your model and return them in a list. | `List[Model]`
 
 ### Bringing your own envelope
 

docs/src/gatsby-theme-apollo-docs/components/multi-code-block.js

@@ -0,0 +1,127 @@
+import PropTypes from 'prop-types';
+import React, {createContext, useContext, useMemo} from 'react';
+import styled from '@emotion/styled';
+import {trackCustomEvent} from 'gatsby-plugin-google-analytics';
+
+export const GA_EVENT_CATEGORY_CODE_BLOCK = 'Code Block';
+export const MultiCodeBlockContext = createContext({});
+export const SelectedLanguageContext = createContext();
+
+const Container = styled.div({
+  position: 'relative'
+});
+
+const langLabels = {
+  js: 'JavaScript',
+  ts: 'TypeScript',
+  'hooks-js': 'Hooks (JS)',
+  'hooks-ts': 'Hooks (TS)'
+};
+
+function getUnifiedLang(language) {
+  switch (language) {
+    case 'js':
+    case 'jsx':
+    case 'javascript':
+      return 'js';
+    case 'ts':
+    case 'tsx':
+    case 'typescript':
+      return 'ts';
+    default:
+      return language;
+  }
+}
+
+function getLang(child) {
+  return getUnifiedLang(child.props['data-language']);
+}
+
+export function MultiCodeBlock(props) {
+  const {codeBlocks, titles} = useMemo(() => {
+    const defaultState = {
+      codeBlocks: {},
+      titles: {}
+    };
+
+    if (!Array.isArray(props.children)) {
+      return defaultState;
+    }
+
+    return props.children.reduce((acc, child, index, array) => {
+      const lang = getLang(child);
+      if (lang) {
+        return {
+          ...acc,
+          codeBlocks: {
+            ...acc.codeBlocks,
+            [lang]: child
+          }
+        };
+      }
+
+      if (child.props.className === 'gatsby-code-title') {
+        const nextNode = array[index + 1];
+        const title = child.props.children;
+        const lang = getLang(nextNode);
+        if (nextNode && title && lang) {
+          return {
+            ...acc,
+            titles: {
+              ...acc.titles,
+              [lang]: title
+            }
+          };
+        }
+      }
+
+      return acc;
+    }, defaultState);
+  }, [props.children]);
+
+  const languages = useMemo(() => Object.keys(codeBlocks), [codeBlocks]);
+  const [selectedLanguage, setSelectedLanguage] = useContext(
+    SelectedLanguageContext
+  );
+
+  if (!languages.length) {
+    return props.children;
+  }
+
+  function handleLanguageChange(language) {
+    setSelectedLanguage(language);
+    trackCustomEvent({
+      category: GA_EVENT_CATEGORY_CODE_BLOCK,
+      action: 'Change language',
+      label: language
+    });
+  }
+
+  const defaultLanguage = languages[0];
+  const renderedLanguage =
+    selectedLanguage in codeBlocks ? selectedLanguage : defaultLanguage;
+
+  return (
+    <Container>
+      <MultiCodeBlockContext.Provider
+        value={{
+          selectedLanguage: renderedLanguage,
+          languages: languages.map(lang => ({
+            lang,
+            label:
+              // try to find a label or capitalize the provided lang
+              langLabels[lang] || lang.charAt(0).toUpperCase() + lang.slice(1)
+          })),
+          onLanguageChange: handleLanguageChange
+        }}
+      >
+        <div className="gatsby-code-title">{titles[renderedLanguage]}</div>
+        {codeBlocks[renderedLanguage]}
+      </MultiCodeBlockContext.Provider>
+    </Container>
+  );
+}
+
+MultiCodeBlock.propTypes = {
+  children: PropTypes.node.isRequired
+};