From dbe131a89f2a2005f61c4934d4d9da9c65680f5c Mon Sep 17 00:00:00 2001 From: Noman Date: Thu, 10 Jul 2025 03:40:13 +0300 Subject: [PATCH] feat: Add interactive Python dashboard for Kafka health visualization - Add Plotly Dash-based dashboard for visualizing Kafka analysis reports - Include real-time health score gauge with color-coded indicators - Provide interactive charts for health checks and topics distribution - Add sortable/filterable health checks table with recommendations - Support auto-refresh for continuous monitoring (30-second intervals) - Include cluster overview with broker and controller information - Add comprehensive documentation and setup instructions - Provide integration scripts for combined analysis + dashboard workflow Features: - Health score gauge with visual indicators - Interactive bar charts for health check status - Topics distribution pie charts - Detailed health checks table with filtering - Cluster information cards - Auto-refresh capability for real-time monitoring - Responsive design with Bootstrap components Files added: - examples/dashboard/requirements.txt - Python dependencies - examples/dashboard/src/kafka_dashboard.py - Main dashboard application - examples/dashboard/src/data_loader.py - Data loading utilities - examples/dashboard/src/components/charts.py - Chart components - examples/dashboard/run_dashboard.py - Dashboard launcher script - examples/dashboard/README.md - Dashboard documentation - examples/dashboard/integration/ - Combined workflow scripts This enhancement provides visualization capabilities for the generated reports without modifying the core analyzer functionality. --- dashboard/README.md | 321 +++++++ dashboard/__pycache__/app.cpython-312.pyc | Bin 0 -> 16110 bytes dashboard/app.py | 339 +++++++ dashboard/assets/custom.css | 116 +++ .../__pycache__/charts.cpython-312.pyc | Bin 0 -> 12602 bytes .../__pycache__/layout.cpython-312.pyc | Bin 0 -> 12121 bytes dashboard/components/charts.py | 371 ++++++++ dashboard/components/layout.py | 257 ++++++ dashboard/requirements.txt | 7 + dashboard/run_dashboard.py | 177 ++++ dashboard/start_monitoring.py | 132 +++ .../__pycache__/data_loader.cpython-312.pyc | Bin 0 -> 11531 bytes dashboard/utils/data_loader.py | 223 +++++ .../kafka-analysis-1752105684783.json | 698 ++++++++++++++ .../kafka-analysis-1752105715380.json | 698 ++++++++++++++ .../kafka-analysis-1752105730445.json | 698 ++++++++++++++ .../kafka-health-checks-1752105684783.csv | 16 + .../kafka-health-checks-1752105715380.csv | 16 + .../kafka-health-checks-1752105730445.csv | 16 + .../kafka-report-1752105684783.html | 850 ++++++++++++++++++ .../kafka-report-1752105715380.html | 850 ++++++++++++++++++ .../kafka-report-1752105730445.html | 850 ++++++++++++++++++ .../kafka-summary-1752105684783.txt | 77 ++ .../kafka-summary-1752105715380.txt | 77 ++ .../kafka-summary-1752105730445.txt | 77 ++ kraft-config.json | 15 + src/cli.js | 27 + 27 files changed, 6908 insertions(+) create mode 100644 dashboard/README.md create mode 100644 dashboard/__pycache__/app.cpython-312.pyc create mode 100644 dashboard/app.py create mode 100644 dashboard/assets/custom.css create mode 100644 dashboard/components/__pycache__/charts.cpython-312.pyc create mode 100644 dashboard/components/__pycache__/layout.cpython-312.pyc create mode 100644 dashboard/components/charts.py create mode 100644 dashboard/components/layout.py create mode 100644 dashboard/requirements.txt create mode 100755 dashboard/run_dashboard.py create mode 100755 dashboard/start_monitoring.py create mode 100644 dashboard/utils/__pycache__/data_loader.cpython-312.pyc create mode 100644 dashboard/utils/data_loader.py create mode 100644 kafka-analysis/kafka-analysis-1752105684783.json create mode 100644 kafka-analysis/kafka-analysis-1752105715380.json create mode 100644 kafka-analysis/kafka-analysis-1752105730445.json create mode 100644 kafka-analysis/kafka-health-checks-1752105684783.csv create mode 100644 kafka-analysis/kafka-health-checks-1752105715380.csv create mode 100644 kafka-analysis/kafka-health-checks-1752105730445.csv create mode 100644 kafka-analysis/kafka-report-1752105684783.html create mode 100644 kafka-analysis/kafka-report-1752105715380.html create mode 100644 kafka-analysis/kafka-report-1752105730445.html create mode 100644 kafka-analysis/kafka-summary-1752105684783.txt create mode 100644 kafka-analysis/kafka-summary-1752105715380.txt create mode 100644 kafka-analysis/kafka-summary-1752105730445.txt create mode 100644 kraft-config.json diff --git a/dashboard/README.md b/dashboard/README.md new file mode 100644 index 0000000..a56935d --- /dev/null +++ b/dashboard/README.md @@ -0,0 +1,321 @@ +# 🚀 Kafka Dashboard + +Interactive web dashboard for monitoring and analyzing Kafka cluster health and performance metrics. + +## 📊 Features + +- **Real-time Health Monitoring** - Live cluster health score and status indicators +- **Interactive Charts** - Plotly-powered visualizations with hover details and zoom +- **Health Checks Analysis** - Detailed breakdown of all health check results +- **Topics Overview** - Partition distribution, replication factors, and topic analytics +- **Consumer Groups Monitoring** - Active/inactive groups and member status +- **Broker Information** - Cluster metadata and broker details +- **Auto-refresh** - Automatically updates every 30 seconds +- **Responsive Design** - Works on desktop, tablet, and mobile devices + +## đŸŽ¯ Screenshots + +### Main Dashboard + +- Health score gauge with color-coded status +- Metrics cards showing key cluster statistics +- Distribution charts for topics and consumer groups + +### Health Checks + +- Comprehensive table of all health check results +- Color-coded status indicators (✅ Passed, ❌ Failed, ⚠ī¸ Warning) +- Recommendations for each check + +## 🛠ī¸ Installation + +### Prerequisites + +- Python 3.8+ +- Kafka analysis reports (generated by the main analyzer) + +### Quick Start + +1. **Install dependencies:** + + ```bash + cd dashboard + pip install -r requirements.txt + ``` + +2. **Generate analysis data (from parent directory):** + + ```bash + cd .. + npx superstream-kafka-analyzer --config kraft-config.json + ``` + +3. **Run dashboard:** + + ```bash + cd dashboard + python run_dashboard.py + ``` + +4. **Open browser:** + ``` + http://localhost:8050 + ``` + +### Advanced Installation + +```bash +# Install with auto-dependency installation +python run_dashboard.py --install + +# Run on different port +python run_dashboard.py --port 8080 + +# Use custom data directory +python run_dashboard.py --data-dir /path/to/kafka-reports + +# Run in debug mode +python run_dashboard.py --debug + +# Bind to all interfaces (accessible from other machines) +python run_dashboard.py --host 0.0.0.0 +``` + +## 📁 Project Structure + +``` +dashboard/ +├── app.py # Main dashboard application +├── run_dashboard.py # Launcher script with dependency checking +├── requirements.txt # Python dependencies +├── README.md # This file +├── components/ +│ ├── charts.py # Chart generation components +│ └── layout.py # UI layout components +├── utils/ +│ └── data_loader.py # Data loading and processing utilities +└── assets/ # Static assets (CSS, images) +``` + +## 🔧 Configuration + +### Data Directory + +The dashboard looks for analysis files in these locations (in order): + +1. `./kafka-analysis` (current directory) +2. `../kafka-analysis` (parent directory) +3. `../../kafka-analysis` (grandparent directory) +4. `~/kafka-analysis` (home directory) + +### File Format + +The dashboard expects JSON files with this naming pattern: + +``` +kafka-analysis-{timestamp}.json +``` + +## 📊 Dashboard Sections + +### 1. **Metrics Cards** + +- Total Brokers +- Total Topics +- Total Partitions +- Consumer Groups +- Active Groups +- Average Partitions per Topic + +### 2. **Health Score Gauge** + +- Overall cluster health percentage +- Color-coded indicator (Red < 70%, Yellow 70-90%, Green > 90%) +- Delta from target (90%) + +### 3. **Health Checks Summary** + +- Bar chart showing passed/failed/warning counts +- Interactive tooltips with details + +### 4. **Topics Distribution** + +- Pie chart of user vs internal topics +- Total count in center + +### 5. **Partitions per Topic** + +- Bar chart showing partition distribution +- Excludes internal topics for clarity + +### 6. **Consumer Groups Status** + +- Pie chart of active vs inactive groups +- Helps identify unused consumer groups + +### 7. **Replication Factor Distribution** + +- Bar chart showing RF distribution across topics +- Helps identify replication inconsistencies + +### 8. **Cluster Information Card** + +- Vendor information +- Cluster ID and controller +- Last analysis timestamp +- Broker details with badges + +### 9. **Detailed Health Checks Table** + +- Comprehensive table of all health checks +- Sortable and filterable columns +- Color-coded rows by status +- Recommendations for each check + +## 🔄 Data Refresh + +### Automatic Refresh + +- Dashboard auto-refreshes every 30 seconds +- Loads the latest analysis file automatically +- Updates timestamp in top-right corner + +### Manual Refresh + +- Click "🔄 Refresh Now" button +- Immediately loads latest data +- Useful for testing or immediate updates + +## 🎨 Customization + +### Themes + +The dashboard uses Bootstrap themes and can be customized by modifying: + +- `external_stylesheets` in `app.py` +- Custom CSS in the `app.index_string` + +### Charts + +Chart appearance can be modified in `components/charts.py`: + +- Colors and styling +- Chart types and layouts +- Hover templates and annotations + +### Layout + +UI components can be customized in `components/layout.py`: + +- Card designs and layouts +- Metrics and badges +- Table styling + +## 🐛 Troubleshooting + +### Common Issues + +**No data available:** + +```bash +# Generate analysis reports first +cd /path/to/kafka-analyzer +npx superstream-kafka-analyzer --config config.json +``` + +**Dependencies missing:** + +```bash +# Install required packages +pip install -r requirements.txt +# Or use auto-install +python run_dashboard.py --install +``` + +**Port already in use:** + +```bash +# Use different port +python run_dashboard.py --port 8080 +``` + +**Permission denied:** + +```bash +# Make launcher executable (Linux/Mac) +chmod +x run_dashboard.py +``` + +### Debug Mode + +Run with debug flag for detailed error information: + +```bash +python run_dashboard.py --debug +``` + +## 📈 Performance + +### Optimization Tips + +- Dashboard is optimized for files up to 100MB +- For large clusters (1000+ topics), consider: + - Filtering internal topics in visualizations + - Pagination for large tables + - Caching for historical data + +### Memory Usage + +- Typical memory usage: 50-100MB +- Scales with number of topics and historical reports +- Auto-cleanup of old data stores + +## 🔒 Security + +### Network Access + +- Default binding: `127.0.0.1` (localhost only) +- For network access: use `--host 0.0.0.0` +- Consider reverse proxy for production use + +### Data Privacy + +- All processing happens locally +- No external API calls (except CDN for fonts) +- Analysis data stays on your machine + +## 🤝 Contributing + +### Development Setup + +```bash +# Clone and setup +git clone +cd dashboard + +# Install dev dependencies +pip install -r requirements.txt + +# Run in debug mode +python run_dashboard.py --debug +``` + +### Adding Features + +1. Create new components in `components/` +2. Add data processing in `utils/` +3. Register callbacks in `app.py` +4. Update layout as needed + +## 📄 License + +MIT License - see parent project for details. + +## 🆘 Support + +For issues and questions: + +- Check troubleshooting section above +- Review console output with `--debug` flag +- Ensure analysis files are generated correctly +- Verify Python version (3.8+ required) diff --git a/dashboard/__pycache__/app.cpython-312.pyc b/dashboard/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a576f1c2f7b010df30c140938e246c892ece02bc GIT binary patch literal 16110 zcmch8dvF^^dgtK%8UTDid;kPF6h+CP1d^gG+Xz>os9_QuIbZ1nWw>#w`N?*8?!d;ZSrbyM)1u;&(jaDbxz7rtnZOZ9C0 zHxor&r&wy1Voj_$ZJIT~uO)5Dm}kuy%d911owa6cv$l+V)}C?9I!sDB=d6pQcV|4a zGzqh&y&3PUSB>}0`bfNg*3bDD8Limb?_y8QulYD%-ITpreC_vNopMi2mgeIEA3|H; zv)wPLn=QQ~(=ywVX`O8)@Ah6q=vglEGh%1ljAtn+P(b$w`7 zWY2bTU99{3QxwIwnJKRGLnWA+WjJPWRDCi0Xz?a$*}QCycnXW|Q}I-mIT=r$k27O& zVPP&8=UL{3rKNN#5f@Xrta~yma(p}?rY>*{tEWijcqWs}rok#9wD3o=(rj6X#{y zj2IWWh)H&`@GGV=T*NHk#rwOFY8TA?I9y>jH?$t9dF*t&b)xtNwV(JkXhiol{CgIKA3@65g-phJkEoMYa ziD~ZGmf2urKnZ><`J27T_=Qv=m#rh}&0c7nI^iRv+K(%RD$M2BW#-MtiqW*gE{H0v zJ!HKUfu&1~kV~gnre|>Q&|Gqpdd!!QE94V6 zn>|79(Bb%@1J4+%{bl5)vdP>Ngzi5SADTP-RfS%R^I2R6wr_-FGO>T~&{vf!aA02S zlbiQo;?OgP*k2)eWT1}WjSM_KLPpREAGgmpdtoMy!BW95*R$$mE(gQ<@oZ=%w2Ugg4 zEKOEap2iSw0@u#ioU(ZGIHk&sFp0s68G~t0TQaSTYEC#FlTtc>s z3tWa1WY>$QPoJAPclL!dvIW{trc+RpOEC2zXpm`v6Z11YG8~L&WpLFfvFtio&iJ+Wmra>K4t!>&rh>s6a& zr?2Xy`uZPw9RW88bXEi5ZhF;v)p^Bv-x;nRqg>vyv%TbOzwZoHoOIdQQF3-fN+YIt-|VX$#^uIW**7%Rp&4YOrAu zt!m4|n(mlCRCRnZgkUYKm9^cmYcRl&S_c<3;jE*s+V+cpAdhag-X& z!$`zFidgsv{K!r%k?hHWNh%qHM{dP{akM`2FX+HGiUH?9zAc%~r`YKT&3B*_*C><> z8gM+1*a`+U&jFN<;f#mSS$*ZXB#t2%*D$r^(gKfmAzQ{$7kIQL*-6$}&=?_k4kjT4 zRkB^FW5mSwz^j1$$nd+s<-5TtgtiPI<2awf4Br9gE$Xvi_-6D*^i#T_Oz$kwJInN7 zi5`^bp{kwoHfmwRC3;w*M_?d%8?PR}a{N8teR>bRHC&y#GF55bQ)y=^9pQ%#tJn9? zhryFqCg1D0Pw##}`_W%)@4HX$t?t+9=$t3wEF0I3>~$ICbQ+y(k*(nM8GSnZ^ABHSIjGx6>HKm zLq*Jmu9r!3z+586nIxafz+577g2?b30$ZWb*!ya;eA=k|XnHXkX{N63)pp91ll#=UMxuR{zP(h4O6ub957G0jkrr;g#Q=tK~7 zwW+XI?P{`isEY(M7Z*5|$z_@Ng?K6*pGza-BF151AX9jYI%j8dq-kS-umnIla+?~i z4?8lOv*wY{L|*va`RxRdGKyfY91-lBcw;Be{3a-3y&y-N1NiY8KLq$=pRF8 z5*z_L>J3I+Z+5~z`=&`s|iTZ1*M{{h|j?!Gmkv-clo!?{)_xHZusWGFZ`y)tzphdo zECK|y8hHNAN^!Fumfo(Ed7wt`_%x_sXumonpCR^N*SdbTfox>(UEe{(sYI7%~S=zile?57Q zU_00_+nF-m>C$uRa{(;QOrd*}?3La=^YdIGFshz^99Pai zW-zWWc=`gIpQO{;smKz?Ctz)cg}b)iF9ER+D}Q2v19`OfIRFO|k--6!K|Nc+?D>2^@+< zfujaO22eJz&4uQ3B#;?12DT#W!dws-vKs{Uno+VAteLs!fghV>7de}l1`6m=!xE53 zDZ*Sw36WF?oY6xVs69hUC`64VppLbCOgzEofcng1sY*$|L_qLp2Q)v_8>m<0aF+zN zfR<4;n3*8K44xk4g~HAklw&KRxO#-8NnjKSkJ`FNzCR} z*}t_022qS6O3MKa4FHL+SLwUuH@D>)QccpNpPH&xrWrOF;-bsO}mYAxbW6&4WN;4TIp zh0B6r@M`d()-a%96|Zg#Ia9y+Opz|~7vE*a>QLnZpqx&i=trmJf2(!SpNCic_}@ zKcwj~>Z`H1we0XE?ef_*eF6KtX|&ktW$ZpOfw5%6LX}u3#U;UOZBUvIUS7AH&x-(82DN z+c!fPg3DrLw;*?Hj#5y)hP>pU8mnR%z<)YK*Ktt|FvkE^D@acltEsO+I?@_)mnznP zRAUNKjVTE<%(WSfDTi1*^0JX8o?tE)mst{*V=%wTUd0O~;!zi$!isw|&<6Y^kE9UK zW3YdVTKsto)$p6rFg^{_@?F;h zX4ePKTh3x+L}HG<>#ekPuN}FaT5qqkGHa3BudSZ~#@+0Yk#S0f4S>asq54yNvZ4Q zBHgi7L`(|51|pbUWoD?v43(L)CFX2#=2eNAEz)72aZ!Z^=bEt0*H2?O<-BaI-94;#<-ibI2b>sU z5dv*_SbecvvBF8Hs;wc>i(h+$RK11$oGwM<-yYP@!5aXF8+3|ArJRJRIW%~uVPL|{I2+*rHul@n^ z8{~bM$zy%0H^1|JW>gh9IX2A5ZXF7+YKm*um{H}rNScESl#oJy%j3nF>do(co0)>+ zedYzc{Vk*f5RP!Ddh-W=^iS(ofdCYNLVS_1P!K13MGS~3JdHpu!)StTsmYizg(Dl5 z9cr?ILlMBp_H2&jCRy2~-xe&mGF)_s0djH(IaG4flRpOUO5cd$k!v8M%u7V@mRc&ytQJ{mu9bJa^LfY>(ke!*PgxWk^;l4u1aI*dgryyHRtV!)OcXk{+YiS zd7cM!!`1OCAU30xRu{Qp}iCZUbUs`XMc8wN$#)`q0DqVed&;N_X)HrJT z!U9CL*-+!rqgIYKyKIxv za0h9d6b6oEn-p+ZsFlXjKh#%k6ZPGc6%V!R9OEt`-Ld6afn!BY4s7;(h0v(N!YKO% zM#b|C;fD*|u+dW^lv58aTHp8;f+AeTk7n|40f`}AJePqJho$HNeg+Dc?J#x1IRea` z(EU+;SwDtJCA`Rs5z9Ifo!^FicRq_}UScUC9e%2FWBl)TFW%6BJy} zF{-`8Q53G0+7%o$SJ|x^R3^S8U=6_<6kGUXcqw#kS@H02$!LLx&;a@%B{6{3o=VHk zO6#6Vpc@*y#l7l=E^P>2{p~Bi{a$kIyyPD$I){|TMHAN^oWx1ven*?F;dkq!p+l45 z>nGNY!gPs?4j9aT7ki>9l@s2~otne|RGvX>Vp3P^; ziMf6>mVXK|Rd4>@TX-G|XTth-f@At;gmBjlzp#Q2tR9($^SA{C@c1qG5>bpZ@QE9T z$Fa+R9=jn%wQJ@TrXNa)>e2dZaBBrT)kOFNs z13nx?5BW|(wsV(&suPrJ(r!MVjlmfohDy%RTJLRkJy3Lp{@!_vjL>Pg)Y#bHZ~V*xfPcht zE`@DTaP1#D)ISLS_D6g?veB|T7R%(=d>TI*p<}V%%E!~{CnSzojLjutG5!Xm<9pG; z{^xh1gNL%DCwPR|gp%ZO>f!$qoF7v}7iB_1)8)}YidJ|YoVTfeu=ag!-EZq!Yu=#X z=ZmDN)z@5+JXLlr|s#70k7>jML3GYuSLkk z$5>=1|5p&eBQH;`%qx2fTmY0yj1Kf6+>_gJWW?s+gB{pt#Fqg2Yv0l+yN$q7Iwz)= z`{yBYA(oq41oIbU@1|E?L~(ebj{kF%X%UtM{$qUoBYcI60`PGmT##7DH&8N}A`RRK zmpjrzKUwY7PknHv;Bj$)LwQ}1eubqZ-vKG7|Heq=)<5|mh(}mST36{!mpcDsqoJz z*Uu<700oov9p|^4@A$vvf7guX(AWLf{A=b99Jd@F_;2~|nm=}Y#;`_$C`rpA9x9R`{I$9~CbdhT-y-RcDwWu;dO|HIP$4-K7C1M}8o#p-?M)VEH3 N`*hVtS%XCB{||`@S`GjJ literal 0 HcmV?d00001 diff --git a/dashboard/app.py b/dashboard/app.py new file mode 100644 index 0000000..2dde7fb --- /dev/null +++ b/dashboard/app.py @@ -0,0 +1,339 @@ +""" +Main Kafka Dashboard Application +Interactive dashboard for monitoring Kafka cluster health and analytics +""" + +import dash +from dash import dcc, html, Input, Output, callback, State +import dash_bootstrap_components as dbc +import plotly.graph_objects as go +from datetime import datetime +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from utils.data_loader import KafkaDataLoader, HistoricalDataProcessor +from components.charts import ChartBuilder, MetricsCards +from components.layout import LayoutComponents, TabsLayout + + +class KafkaDashboard: + """Main dashboard application class""" + + def __init__(self, data_dir: str = "../kafka-analysis"): + self.data_dir = data_dir + self.data_loader = KafkaDataLoader(data_dir) + self.chart_builder = ChartBuilder() + self.layout_components = LayoutComponents() + + # Initialize Dash app + self.app = dash.Dash( + __name__, + external_stylesheets=[ + dbc.themes.BOOTSTRAP, + "https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&display=swap" + ], + suppress_callback_exceptions=True + ) + + # Set custom CSS + self.app.index_string = ''' + + + + {%metas%} + Kafka Dashboard + {%favicon%} + {%css%} + + + + {%app_entry%} +
+ {%config%} + {%scripts%} + {%renderer%} +
+ + + ''' + + self.setup_layout() + self.setup_callbacks() + + def setup_layout(self): + """Setup the main dashboard layout""" + self.app.layout = dbc.Container([ + # Header + self.layout_components.create_header(), + + # Refresh controls + self.layout_components.create_refresh_controls(), + + # Main content area + html.Div(id="main-content"), + + # Auto-refresh interval + dcc.Interval( + id='interval-component', + interval=30*1000, # Update every 30 seconds + n_intervals=0 + ), + + # Data store + dcc.Store(id='kafka-data'), + dcc.Store(id='historical-data') + + ], fluid=True) + + def setup_callbacks(self): + """Setup dashboard callbacks""" + + @self.app.callback( + [Output('kafka-data', 'data'), + Output('historical-data', 'data'), + Output('last-updated', 'children')], + [Input('interval-component', 'n_intervals'), + Input('refresh-button', 'n_clicks')] + ) + def update_data(n_intervals, refresh_clicks): + """Update data from latest reports""" + # Load latest report + latest_data = self.data_loader.get_latest_report() + + # Load all reports for historical analysis + all_reports = self.data_loader.get_all_reports() + + # Current timestamp + current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + return latest_data, all_reports, current_time + + @self.app.callback( + Output('main-content', 'children'), + Input('kafka-data', 'data') + ) + def update_main_content(kafka_data): + """Update main content based on available data""" + if not kafka_data: + return self.layout_components.create_no_data_message() + + return self.create_dashboard_content(kafka_data) + + def create_dashboard_content(self, kafka_data): + """Create the main dashboard content""" + # Extract data summaries + health_score = self.data_loader.get_health_score(kafka_data) + topics_summary = self.data_loader.get_topics_summary(kafka_data) + broker_info = self.data_loader.get_broker_info(kafka_data) + consumer_summary = self.data_loader.get_consumer_groups_summary(kafka_data) + + # Create metrics cards + metrics = MetricsCards.create_cluster_metrics( + broker_info, topics_summary, consumer_summary + ) + + return html.Div([ + # Metrics cards + self.layout_components.create_metrics_cards(metrics), + + # Charts row 1 + dbc.Row([ + dbc.Col([ + self.layout_components.create_chart_card( + "health-score-gauge", + "Cluster Health Score", + "Overall health percentage based on passed checks" + ) + ], width=12, lg=4), + dbc.Col([ + self.layout_components.create_chart_card( + "health-checks-summary", + "Health Checks Summary", + "Breakdown of health check results" + ) + ], width=12, lg=4), + dbc.Col([ + self.layout_components.create_chart_card( + "topics-distribution", + "Topics Distribution", + "User vs Internal topics" + ) + ], width=12, lg=4) + ], className="mb-4"), + + # Charts row 2 + dbc.Row([ + dbc.Col([ + self.layout_components.create_chart_card( + "partitions-chart", + "Partitions per Topic", + "Distribution of partitions across user topics" + ) + ], width=12, lg=6), + dbc.Col([ + self.layout_components.create_chart_card( + "consumer-groups-chart", + "Consumer Groups Status", + "Active vs Inactive consumer groups" + ) + ], width=12, lg=6) + ], className="mb-4"), + + # Charts row 3 + dbc.Row([ + dbc.Col([ + self.layout_components.create_chart_card( + "replication-factor-chart", + "Replication Factor Distribution", + "Distribution of replication factors across topics" + ) + ], width=12, lg=6), + dbc.Col([ + self.layout_components.create_cluster_info_card("cluster-info") + ], width=12, lg=6) + ], className="mb-4"), + + # Health details table + dbc.Row([ + dbc.Col([ + self.layout_components.create_health_details_table("health-details-table") + ], width=12) + ]), + + # Hidden divs for charts data + html.Div(id="chart-data", style={"display": "none"}) + ]) + + def create_chart_callbacks(self): + """Create callbacks for chart updates""" + + @self.app.callback( + [Output('health-score-gauge', 'figure'), + Output('health-checks-summary', 'figure'), + Output('topics-distribution', 'figure'), + Output('partitions-chart', 'figure'), + Output('consumer-groups-chart', 'figure'), + Output('replication-factor-chart', 'figure'), + Output('cluster-info', 'children'), + Output('health-details-table', 'children')], + Input('kafka-data', 'data') + ) + def update_charts(kafka_data): + """Update all charts with new data""" + if not kafka_data: + # Return empty charts + empty_fig = go.Figure() + empty_content = html.Div("No data available") + return (empty_fig, empty_fig, empty_fig, empty_fig, + empty_fig, empty_fig, empty_content, empty_content) + + # Extract data + health_score = self.data_loader.get_health_score(kafka_data) + health_data = kafka_data.get('healthChecks', {}) + topics_summary = self.data_loader.get_topics_summary(kafka_data) + topics = kafka_data.get('topics', []) + consumer_groups = kafka_data.get('consumerGroups', []) + broker_info = self.data_loader.get_broker_info(kafka_data) + + # Create charts + health_gauge = self.chart_builder.create_health_score_gauge(health_score) + health_summary = self.chart_builder.create_health_checks_summary(health_data) + topics_dist = self.chart_builder.create_topics_distribution(topics_summary) + partitions_chart = self.chart_builder.create_partitions_per_topic(topics) + consumer_chart = self.chart_builder.create_consumer_groups_chart(consumer_groups) + replication_chart = self.chart_builder.create_replication_factor_chart(topics) + + # Create cluster info + cluster_info = self.create_cluster_info_content(kafka_data, broker_info) + + # Create health details table + health_details = self.create_health_details_table(kafka_data) + + return (health_gauge, health_summary, topics_dist, partitions_chart, + consumer_chart, replication_chart, cluster_info, health_details) + + def create_cluster_info_content(self, kafka_data, broker_info): + """Create cluster information content""" + timestamp = kafka_data.get('timestamp', 'Unknown') + vendor = kafka_data.get('vendor', 'Unknown') + + return html.Div([ + html.P([html.Strong("đŸĸ Vendor: "), vendor]), + html.P([html.Strong("🆔 Cluster ID: "), broker_info.get('cluster_id', 'Unknown')]), + html.P([html.Strong("👑 Controller: "), str(broker_info.get('controller', 'Unknown'))]), + html.P([html.Strong("📅 Last Analysis: "), timestamp]), + html.P([html.Strong("đŸ–Ĩī¸ Total Brokers: "), str(broker_info.get('total_brokers', 0))]), + html.Hr(), + html.H6("📡 Broker Details:"), + html.Div([ + dbc.Badge(f"Broker {broker.get('nodeId', 'Unknown')}", + color="secondary", className="me-2 mb-2") + for broker in broker_info.get('brokers', []) + ]) + ]) + + def create_health_details_table(self, kafka_data): + """Create health details table""" + # Extract health check details + health_details = self.data_loader.extract_health_checks_details(kafka_data) + + if not health_details: + return html.Div([ + html.P("No health check details available", + className="text-muted text-center p-4") + ]) + + # Process for table display + table_data = [] + for check in health_details: + table_data.append({ + 'status': self.get_status_emoji(check.get('status', 'UNKNOWN')), + 'name': check.get('name', 'Unknown Check'), + 'message': check.get('message', 'No message'), + 'recommendation': check.get('recommendation', 'No recommendation') + }) + + return self.layout_components.create_data_table(table_data, "health-table") + + def get_status_emoji(self, status): + """Get emoji for status""" + status_map = { + 'PASSED': '✅ PASSED', + 'FAILED': '❌ FAILED', + 'WARNING': '⚠ī¸ WARNING', + 'INFO': 'ℹī¸ INFO' + } + return status_map.get(status, '? UNKNOWN') + + def run(self, debug=True, port=8050, host='127.0.0.1'): + """Run the dashboard""" + # Register chart callbacks + self.create_chart_callbacks() + + print(f"🚀 Starting Kafka Dashboard on http://{host}:{port}") + print(f"📁 Data directory: {os.path.abspath(self.data_dir)}") + + # Check if data directory exists + if not os.path.exists(self.data_dir): + print(f"⚠ī¸ Warning: Data directory '{self.data_dir}' not found") + print("💡 Run the Kafka analyzer first to generate reports") + + self.app.run_server(debug=debug, port=port, host=host) + + +if __name__ == "__main__": + dashboard = KafkaDashboard() + dashboard.run() diff --git a/dashboard/assets/custom.css b/dashboard/assets/custom.css new file mode 100644 index 0000000..a361921 --- /dev/null +++ b/dashboard/assets/custom.css @@ -0,0 +1,116 @@ +/* Custom CSS for Kafka Dashboard */ + +/* Global Styles */ +body { + font-family: 'Inter', sans-serif !important; + background-color: #f8f9fa !important; +} + +/* Card Borders */ +.border-left-primary { + border-left: 4px solid #007bff !important; +} + +.border-left-success { + border-left: 4px solid #28a745 !important; +} + +.border-left-info { + border-left: 4px solid #17a2b8 !important; +} + +.border-left-warning { + border-left: 4px solid #ffc107 !important; +} + +.border-left-secondary { + border-left: 4px solid #6c757d !important; +} + +.border-left-danger { + border-left: 4px solid #dc3545 !important; +} + +/* Dashboard specific styles */ +.dashboard-header { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + border-radius: 10px; + padding: 2rem; + margin-bottom: 2rem; +} + +.metric-card { + transition: transform 0.2s ease-in-out; + border: none !important; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important; +} + +.metric-card:hover { + transform: translateY(-2px); + box-shadow: 0 6px 12px rgba(0, 0, 0, 0.15) !important; +} + +.chart-card { + border: none !important; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important; + border-radius: 10px !important; +} + +/* Status indicators */ +.status-passed { + background-color: #d4edda !important; + color: #155724 !important; +} + +.status-failed { + background-color: #f8d7da !important; + color: #721c24 !important; +} + +.status-warning { + background-color: #fff3cd !important; + color: #856404 !important; +} + +.status-info { + background-color: #d1ecf1 !important; + color: #0c5460 !important; +} + +/* Loading states */ +.loading-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(255, 255, 255, 0.9); + display: flex; + align-items: center; + justify-content: center; + z-index: 9999; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .dashboard-header { + padding: 1rem; + text-align: center; + } + + .metric-card { + margin-bottom: 1rem; + } +} + +/* Animation for updates */ +@keyframes pulse { + 0% { opacity: 1; } + 50% { opacity: 0.7; } + 100% { opacity: 1; } +} + +.updating { + animation: pulse 1s infinite; +} diff --git a/dashboard/components/__pycache__/charts.cpython-312.pyc b/dashboard/components/__pycache__/charts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0ec6fa3eba050e0db9b0bb0d5aa4d40fc802004 GIT binary patch literal 12602 zcmd5?dvF`adA|b=PXZtb@B#37B1MrfDN@wKiWHfaL`v2>7Ae~?V;er=4j@Djp!NVs zgh|JfI5U*vR+5uObS#gVwwY2RWeU%9T4ttgE0q^b2! z`h9x`@JLaz{a+8{-tOMFkK5h*eZTM9{khxiWZ?QAOLElT%P{|miQ>iSkvl1fTwp|I zm=Rgg6lcdx!)8{CbK{m_D~+4uwsHHgUC(z6J3*FCas0x3YuqwJbNjxa?%uZWWMI?9~DT74=2Ybk_jo1lKDuI@CSv+n7|JR z^5{rXAfmHBnMet-gv<-^I3K1ZqEbR4LMoO_@R4*Pj1ien%dte1e=?p-#V4INvGpTC zR&|dHV^T;?k4(S=ImoHpKrEb6xkE8IrJ8poCRO`aCa?k_9@cuSg?2r0C(GghNz5<{ zgJcrfVY9@EUeP33M6+ZSImvds!dOs?N31XmmdpBsVF#44N>0ftl|@Ve$bDcrPPDzs zmCCuGoc(i_a~R6Gp`26l(30g^i=WYInW1zAly-fg(r!cP$}ilu$57e}rOPGX@s9h) z^dVzbVW^>cvNGsXU36{jPRHV+L^2(Egb&9BSzBgdBEbqOtr%@($*OBq65^@RkQ`1D zNwuV6sko$CiIhr{L|9+bv};umn&JP>X1YM8;F}qHj$w#dG)eGzV#uXqh&p zKvKJo8*9?@3{Q(+XHgvPq7`biiT0>T4;>dxQ>^F|%VMnPx(EwXbc-Ib{GwT`5Gz5- z4vW+Wsj4XxNK}h8Vl9;P!;^qm2l9T=4DoszH;XL98)%%Rr5iCG<-{hjS>!=_4b8*L;zflWz5~ZZ9%i@AW}pk zQX(v=b|S@Ld7PBgvXeqQEdel%3#lVFMJTP>MS+ZsKrWQA5FrtjR7WZn9y=8iQ=?im zoQx-lYSC|}Mq?@ICiY0>qC}Xyxe=lnTWjP?;#N@znn*LGNsFEtiEIjs@Ko%T!jWE# zY?ee(P|ZXVZ{iSMGErI8LK}=i4qU%~9D2jWV+ricDD);F$+Bvpy${+{PBsQ%GlEBxYbz4v9J(g3_gp3stNghqc#h_K?#*-kt9-30EVsacHnUrC|LdRRR z&^fsI0u1>**ZxFGB3=A05)BswB=Z0MrDARw}JREkAMQv?x8 zbx#Nr5(yQ%=t9^ora;ydO{&(tv1l4TPRIU)7z+z27-)BTLWG4BiVKs;bSlUZ)O@si zTHEGGEGpyB@o?RX_B)8j(o1k@#u#(D-Mc`TKj=sA=O3-KYSRftL(d#E;A zFMxWO%t)kX43*PTQY>nzo+TyLLv@tgJu#^UD9A`5Bc_DtIAzuu@u~ZPa;RElP{_uQa#aXy)fyW}{h0zG}PDyk>6W>}WQSuln-J_@>!dHkhy4QE*u5yaf-_ z-jx;d4PR2$v?-0vw`^RK_g0|N=eg_l2b^~pe|g!Axv-gWxUSm+d3!+d)hP9C*`|Eu zPNgoW1Ud>1#!+{du{fMiSA}l|%D|4ORLErfLd!Uv28{wW~80l62LC;;+bveWDo zbKEFJMFCDtqPa*%9w5PerUWp6VjWTQv>5gl)vuQ)I3Ju0@#GOs_zG0;x={WXbYqDxabsBl2Vcv0bqRS=`$mFmls zeliTeJqK`K18ZJ`dmZeMGxIiX(fF=#|0mrOQf|`~@_|%119PajK1d;${ z4f=V4KIx}rDafm?2o=(@RQoAiG-xK+ZQ{kC5(2=$bSjxK?-=RJwCv~^>Dw_v`udaU zL~1+V_R{3^4$ufh=-bhw-GerQx`I^Gq6uMKQeDu|F+e8@lEjY>T&PDTl5z|=r0O0` zo|H&R8lM2|A!VxfQ8K}6UE_zeH&v}eDIt{xaHfruZBWl%0u#)T?HKI9dL2;`6KOm5 zY^~A;`~{sF;Fg_6EDveaq-=62K_0`BP5|svaS1rGNFkN)u?a`0CF+ zqWJvh+Lh`?rLje+sXzCO(%7o-t#kY4w&&I#%{2|)Ew|Tu{=nGXM5Lrhdl01O}S_Wz?7M4TzL8Kd= zBDCCDOCb_1zNqPe5NQ^)HpiZ-}^1^^lWO6QT#qtb-{2#{nJ9isC} zS&?Z**(hGi!Cw(#Dh*!EPk6ku)_vCt^BH&6FXd;E&O9&y-X81C&9}P|MQrG^Ev_ zVODtz84`%-x$u#1*sygZ5`oC0;jLS?iqY@CiPx_mzvqh!BT~E*Xi*r9< z*7g#8GJPMGQITncwHOWMTkx)5cd=gsDOJFCfb_BXu0*LT2 zkh#RF94@bnPkU#6z%VWp(N+Lj#&Il}65`mW|I$YL#^a-}d4fWg@rAD>&ZF;@u~Q^cmj^Sz6% z^*T1?8rILBSakIkEM|lwEYL9fY_77sgh?N}JZJZw*{jqx;$O92X!}II@YHaTLT-~Ouyb!)NnETp`A65tjvnk*y zlu=}{1F}>Zkj1LK;4!@!2ZrsrXT0bbZu85n?15nc%Udd(Dj z-t>BaW2O<1O!5W;OmNV`imSw;9%Om~c)`yCKMG5t{i5lLqsVVqCLv6j4E+S&gZ&hp zXoEmM9MBIA`(cm)GVs860WMU07bsl=r7hF!!{5O0wCHOxyyFx%Wf9%*4<`MU3IhyQ z79orp-6fb?Dyp$C&EN+p^urGfez6{ma4_&=?3KpLI(TYkc-X|kOii<=AiO^oY|6Ch z05}1@I?8fn9&m)GD`APWsm_>OguT6)sv}A8#}%c};gU{>s)gEi

!FCK@&*0Rb4Z zH5f-5@sdp=RorQ?#8qw*!X;QBlPHE0M2*VjG!nb zZ&vbkh^uUhjA0&Pp3W9RNN{DfqBN*La)BAkGU{jwiM+UB&k3CbZ@4w@RvbgTx7 zt4W@LTID(b3~JZ520yAfID0h9{)6RROE!LW&#yKdc;iH_=HN^j1piT9_ldXaLhyX> zJDoHAK(?;Cn)0qDrMgq8ZkXGuc&lf9^A@FvpWl|R-U4Q$yZH}{1;Eh5^lbmAZu@-Y zZ13#TZ*9xfZJ#*^fwJ@9N6m*8dgs``vHZX?7tikc75`|i`B2W)0HNO0#}(cS*7H`S zW{py_PFd5X1cJ(%cBN$yaJJG5INRa@ob{E@^e-cC9hAHUm!mBhv%vnqHuml)F*hH@ zS0~`BpW^GMw-2*S?IHHl+kU40%b%*vhh+RLzE#^F;NIpScrW1UZ{ptT4D`2hYIztu z-pJKMQIbqg$jbm3L4gADNga^o0fTQ5jL4N&2}}{HbuMI-L7s>PGyD)zf#m_1+Lyo- zN&$ebymSVqw+CRu4lzrHuIPG|K~+&~y98^dCGIxG6mzFc889ZN&B*b9VRC3B!`&j5 zixqIUKK%U*PggUE0cxkLVx{Q4;w!R0V1iY}wkhbZ_I^%yL-4Bt$ktx*8+@Dap20w- zRfo;uO!3-G@iMjjG)p(*2b^3fK(Ug11BR0#Z6pD!t8aQ%^rN9IzXX@Udn!-^T*8;EM$UVmn+{HOP zLHaT~Rzr>Au;?p16vmyO?j3RT&dGh>PoI*S>#LsU9VgoVViMhKl_;U;pjmb0wsyQwt$V(6dq3>WW1!b4k zBMpRAPfYKjmNxWAf)X+=P7R`Uj4oQ#tB&F77WJy4e~-1_g5VYAmYu0+{&-aw?s*^# zgZB<#*u}UjuDcrZuEx2c+=hKQSL27S{eX}&2ku%Ke?3gh+}A&>8(OR#nsI#`XqXKx z2G(a?`M{1D&kdlg2PpjoXXczISJ^XTRlM~Nqr@dXTxLZ+Y)3w9r+m1|U}S>{ZRErE z`5LJqz3k&D(cBJL#rWa;|3TFlkY>cv=erOiVJ~Ji>lmDM#ZMXWb2XVMPV^LIM0W<} z9dMRIchu|M9Am&`)A5o`4G;C}J|wp+E3t|T@xBtDAIFhCuDGTwK-M|k@IXQj8;MgK zTKXEH2a?MH&R3MMUiUtHPOQfrGzaXV`2p_(8l{F95Nn{+4F*aL=&uuL?i3Sj%yj8` zfJhT@+^wLS*6?-1PPfsX)D=QGCcF|~3jya*3Y>!! zI15&$x^~8)RM+2TOpYxxdz8wW3xV^2Tzz+zU98+Vvs-Csd*|?*hv!@7g~f&q+1`A^ z=9&HHMqWEyut0&kcBb5SZrg>v^L>lH)z(D^*#3dHOY!=Z#vt?p{hLh(9`1tJr1-lO{|ewmYV2F#)V!Y>cQU^9 z)DZcp$2Cyy0$&vD(uyH5vJ9b1-pya=y!SZF!pe&b7z}T2!jgTrKFmBjf@fMNZkndY!z0@zN zv)&>TMh<@*>CuV{QBgxgKtkPqpt7rtgi~yUQw0t@R{;3A27WDKE5usGwPG8bLRbg3 zMf-9*4R(~gu=>`Kc20n<0v@-#b@Xfi^c)3rfNcq2i99+edEq4N4hbV*V1FJ;?bUZ| zhVb|Ye+-uyq38*QI@rf4VSIw_Eoa==tlX*jNac*nSdNUt0NuptB|(fK>kP8g&5;=Z zNHxk9eYXR!or1s$?h0^`(&j17Ztw1mOt(E`Yl>{ac8+4u34vl^EFYXu&Ql`TAetbt;@WEJ*9|UQH01gRI|3@=sXc|DZz&2WHa#ckjU{`HJpiaSy zPNpzLH>*am%TU*kuo@)NMw4Up?RdI~KF4-Em8%3~k$n``!8hxwI@|YJ-|E=D=?h`| z0LAt_6x$7(PdDskXWeJq-yOMd{QPmnQ*-X6yr&iKFXuh$luBNyYzEk#-9LZy!^)nI zeYLYkzq>=Jdl7b*YLWFi%BXJwt57@@N=*Z9C^ZBXz8l@UHFp^+xF0GEtazQ`Z+a+1 z^TF)V1#@;^-uqZ#B}`Mw3`g!VRsHcaoF5^)aUBF}RC|aznL;7e84AH+l5|{xxGNMo zkrv|Ueu9)O6cUqR%tkvo79N*Uqe<`_M&d~U-S;VCSPy$~mJ!2MGi;}jQB1y!0b&qA zdPm;C;0G97fj~7)h^hlVHDND-_Q*d%-X(^1oI1A%0u~KWm&jj*iGG#&HMjAGrSg`k z%i_yg3Jly9!t|C43D-lT`OxTX%)Db}tlk1j6`o^hv<(WZ1eL zd(Ym+s4fn%!Vyip8N|8l&_ZkWtM7UW3?#2^{Mokmw%x|4E)KE6ZcY3Vi*J4)Ta^tj zY|4(l+m!FxSzsWq*aSrNG7u}kn_%bKp$Tze^HuZJp{w>E_vIfwP+%bM8u{hq`;)gZ zs+WOSVNfg6O}lk$p>;uC=y>;q{Dy%719{iHzYM$|xQ$W048#gAYGrz$Z@d5Y(4|8K z2BHgtKY8}Y@YWEeqPI*C(*_;KI}HD&tEkKFya-wM9lL}>mvDsArXQg+FH3RDQWnW> zOrs8r*hQ=1n4n|Q20P%c!>|Jmd!_ww(p1iL?vvu^+g|0AAS^_XS^&ftEIqbi#J?IIi!1x*a4R$g%G zuXH2@{coZbWXm2P>oUmV*+G;o2g`7^dMgaF;3=cBz6Z!w8Dy*VdSQEK)p~0Uvi`R? z$|9g2Q7hW9;76;^1a$jJn*iA`0~t5nQsj4&Bpk|+Rd)(^7(*jklG-(CWH5zyzW2lb zUO21TCP)mF<7ClCvmTzQTnr|PZa!pOT6LO{0|z^3ZQ8V^5oHSReCOLB53`-A&_qk# zesG*?;!D2nVv*NWo16}Z;V@pNynngBuGog=8Pp4$|DkHrkFaFiyQp7Eckiigut&gQ zrF+EExORSb@mrayT_@q(b7^yEX0vB#r3!yYGJw$HgH_Fa~g@ zagxkJE)37V!HxVJgI_>^zCHSplupoTI{`Ji$yEFfbg4}1VZ%37w3Jo|JpqeMR-kh5 z|9cob@6~oa5X=XH^F0f_i-E@!{~9IGa?7E)@#{L2K>PJTXFkw5e`I0TVxSKdRiL@_ z9NhEz^ITWoRdzA3n+hyd&w)$8!20>bg0L9ax#Br=p!htwaAGmAOYt`vJFr1% zc;tFRcfO%J*R%gx?_$H@YvEk|AbR@)Yqf4X-sW@8*bvPtYG*vUHguAI!ePYyU-AW2DqfiPcfjHit4Q2VDe`eJb}R;4BjsW12|@A0;5ja0y?Q+>a^6(AIr98ajQ literal 0 HcmV?d00001 diff --git a/dashboard/components/__pycache__/layout.cpython-312.pyc b/dashboard/components/__pycache__/layout.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebe67279584c967168b00f2c6abe593a0f86a66f GIT binary patch literal 12121 zcmeHNNpKw3dG6U4%nA$!gO!bi5C-A^AV^Ug7m)x7K%yXtph5+f1vRF7Fayr!?j9^{ z(AI&ekd_h}u~GzODg=|b1cs;tPPt>=xk$0r+eUS? z$r@5;gF1Vu&b4R^JERHcX=X8#6X|FsmCdBNv?$QA3{Stp#LhGHFeAjLGYroK_7LjY+3qSUF|%GBbpWV+Ro-7+UH$C7DkE4~zixyE z7y;jUpbDjtt<_OB63W`ygHrJkWoGRZ~7 zL?%sVVr6DjSx?81vX$q=9G|Y>EHA9k1pklTA)F;bs{Up*K`EB;scS6?rLI}mYSmS1 zSzjD)+(IL8g%U*T{kVu0;jK=i())uWG&^AuBWd`Pb+JyN?Tin)((`~ z{>+$4IRoohcb4Wh=;~`&*Sh;Q?k@Z$sIW(B7~(lbAtlGlrZbs&*#sQR)q2=vD_t3V*)YzofLGQKhty&LNPUI6&AzBh{s6_@Rf=`2>U!`WK3x*|XnEE#C{RN{=l>e__;I{^eB~XW|LR&YL z#gR3aOFAfc^5|6{M)bpeksr)s!Fh^;&RVO^Pz6Asr;V)8UhV;4<)|n*b)^b0f!;3c z{bP!Ce{9lIu^!gT`quroHBi1_S~ASmtE<(gOC``UP#^rKSr6wyJ)N3gD$NeiRV#%w z=+bzJj)DmUrVog-IT6jIzk)#nyc@=m?9ogz!sNt^Y?)({IZpBeP7Kg`ER!u_LQHZ} zy$)xD6q8IMlnuxx7(oicL^P93%%md;kxK~?73?I}bcP4~j3l|37?CYNn&BlMEyNi% zGanIBbUdpnPRFHMT0D=P~`8^6C#9w#GazEJm5QKftrBnCnf>)nieimWKzHGl2Xt;WM`82|i z{hA#yX!P3XJ-X`?+fCb=b#3mJCx7DX7X7ATGIqD!4$T5%4(CU|wMD;;16XfbSLfDdZhk+1;`g^`#Ix=;SWIs};92W{;;?r#DL$(E=(gY9 zcuX-lJ4)cygy2Q*Q^(9deK>#kbiVaf#In~60A?*7rG*NHR*tU)w_81d{g;N7Hp*Rl z<@Dv#D}i6S+LU&9&+RTm-TBM&Kg_-Vy}Z3u1vz^(&hX+CsGcs|;u@9{qI@EYW>FMz zuJ}XAC%ET>J-n)yf!`kKY{fxm?ZsUAxny7s>&7AyEgAU+u}ohDe?Y!XW%@LDF*C*@ z#^~_4ig_d=dz0U66%%I(JRjDYFs$2(YndrYHlUBaJSWJxLu%IKoT15v5;>>md1Q3{ zF)Ddgupc|i(mF1C*o2S;h5BlS((N>S1_opJQFae zU7|GxRV`JT<-iF~3N+b0!mYVx*fZtEkKU(;NgJ50z+Pb0(nE1BdS2iULA$D*z;{Yz zc`nYSg~S|JVH;N(#Qd|+=azwg9v+gT#LlWpvB8w+i33Bj0u)-*#x$Wg;ER{6E{m?mv-j_h@Wewge7@)-EyJ0Ov*ER&vrSORJL_ygHC&+ut*`*XM#UwpLs zA9t>Rtqx43R$TlARD{j4_4VTu6DNmd>#5^or{QTkcl^xw*!YNS9veUPn(PobU@8`3 z4zl5xI+1}Uvw}6iMFwwaOF&46Fs5k+d`&c_Wy=YCAJEu{PO^)<-IU6o{0mT3sk-Yl zCRAT^icm{kyYZi%DvD#Ntf$zi_TG(WHiOR>=$?(eo5AM_P#)cEI8tcqT8nQ6o_*l3 zgq)XLiifIg``q8Y>2F`{-gxEHy3O8KKl6`Yvi`fb_R8Yr#e%E$hlehYuMDjQkpZ3e z{S7O&&CQsBUCXMss-?O_0h1ChlJ&E-Y#ke5>#y4&;M>54 z*hZKk#5S?bAU-W@Yr?>`v3pn=YFpTLD0P%dolxp3mG(lZTP@9)*{9eZ_zSZ>uta-r z*QoCfV(_%oq=^9oT)vnD<|k-I^tt38&!|Oa4zxUm%B7~V;i?g=1---#Up?!d-%$- z8U0S#nnol~%4XHiJuonkU65R37>(OUALkR;wEtjs0n)+jLf9rda8bw6o0XmOad2oM z`W5A5Yn)5W#6{VjViwL}R84lq;Ev%GlS(8PWjp345)v#g%djj6wq)y!Jr{dE#>nRB zOp=xS`?CwQkVz(3x|8L&LGBQ|xObX~o}b|}xikx5w+ye2c}|_ls=Ch*#KTnpn#{pI z@FH%8E0##Y3ru|{$MK6fba20`O5FtrDQg!WfIzSY_8`ZyjBJ@s!b~uuF%cs}5Rlld z=f@wA)w9pC4AFd51}BJxRI%8BsHz{UP!SlzRqW@YvHf47_`!j|am{hEBQwYGv1Dfc zt#A!Fd;V=~wn3O6mr4t=lQ7&=luIUMmzoFEg?K`-1vya;sx6pD{9+TRv*An~S>Tl^ z0Z*hx$bj$3kRZJ5)^<)S)~Mapdmc+UoIoN7JO+Mc$1se2R=o%!OC+`;|2=4{M9HFq zL)M~7$gU1fnMS~I_5vKeM?HXuW#C?2!`0`OpF>S-UAE>Mdh>_QZ22Yt=~~fA33vhE zz*BKj!M4@6Hs&^6M+^0xYlrR}+H{>L)VHlB?gTeoFBR(d<|9Y$Li4!du+%v(*|t5D zujzAd=cc!_;B6@Q>kEPA`>maYw(dd`4Pnqi+rC0eN1?g>f!7}LC;_S~0=QggiafBH zn>>n(YVIp^_CBzfpwUIO^yZ&Fk#8C*JT*{g?E(^i?GtNH=9^A^m3C)sdwrKj*+(@r z7D9BPsiV-?wHnKZBHL4j0iRQ$1{_WT?5I>qzmgC3cTn}UAp<;7@|#7yyW=O8TdM$Z zCD`j|tAq;x{&oIZ*$103pt}#&jeXs5Tbm7F)+q%wm`!FF1P@xsCemO@O7*9;3iaXv z*`}|rPYM#4)v&b;xvJmP_c~kz2pm1nBt#mp1MWWOIXcFP(Ks!}Ik@%|IYBJ>Q+y9> z1HhJgcR@hrELa|9lpWQ8VUWNX%O#T#wc)sQ*s2;4wh48WIH8dVEw zCWQ8aQ^gm2YBwul2jS0PIN>cwieA)rFMs!*%Xek;GI5(A4v-J^=U<%I@|`V-4j8fL zw|q%3VuNZJKrvZboFF>>`jxS@(Yt}|8jEklpimaah~cZ8g&|Tj*S^{9TbXyNaaPH4 ztrm>h!C7V6s+?6eGjzsDmB3T+mRCoGX5fw=@*@PAbbh>!zRLW%)+*sFxUSb$51g(K zom*Hie%4(@3maP96_Eo^DRPkBX;aXezRv|==@2>i0S#scVlsWaWOlfjB*%*p)DIv% zsu&ytGF56yw`x3>1yz6_41mJzqcu6+M#S1Drmf!{P1*tj8(*0RRjQ5L~9hX*4 z=|CFQc@E-k5O3y~RHX7&i;hH8YZ~=2MA`agg-n__iqgPYh-kn~a15`^U}CuKD_w!n zVuqeeaPwq6(4I?*LZ9RyQTdb@8RW-d52PBM^z-Mi1~2`dfxsQID+z~dR{n9^7QQT$ z7<`dUuk)N50NF)G)0ruvpR|pac9qc`gntjC3x5fTDx-lbTC8sD@)*h_k+e;ow3{q;!nBL_A^l%o(BECj|td?7GxrP{h6fbHK4p*$c1)rc2) zFE78G-}A<2fiq;7`>j3T0{DBCdUu0IX`+HHs~5JL-R?t*$qRA4HmKO%CY>q+}w^MM{1+xYRpFi{p^E zEXi}40lk~cf^c!{3$z*mA{TKk_*y`WXf0B0S)*gYf+Z!i=)ZG#D=?}R zSNrmZU*8J6K`;hx3UC+;3-Zz$wc2B={F_WqD>K%qj80b(nRhYd0n zZk6p*Q|P`@DK--!2~6pTwd~nm*weT*Bm!F!^IlFv!jS2_`F; zRGu!6YMvx0wZaM@+ePYE=E3_GzhbPlgjWvSICAaCLkhBwOq8{jR4P3r7qqmkvTFx7 zOdAs$_FG3cyI)W!sJkQl!~8GiA7WPP0y)@3YwZK0)6%m#q)_nKFpS=^dSXVtsQboH)(FN46IIk za_;82hZJN<^8+K~RD&ANt3q=j8{qE|3(4>B1)`C&&sO;~0=`u=>%Mh!E?UA?*$FPO zph~~gtUk$ynBe$MLzpOm0hSm9PIzRhiq+!sam9~3zq6j*)pD+)-H1jleSGp{O^7~P z**^UmTF|}Um=>+z(i&zg5V^+>LFfeSMA?o(6wKkr zT@ZrflCl*}0B(G8c#v>q@srB?HH>57bz2UPh1|oP&1B*8Bj`q+IfNtS#_~_-p-fuH zL0FC^ib8--+O!Ix*!L2BmWSvY_9ag+RAJE(m|pf2b@a=oM# zS0WqsE!Xp<#(_J{TdtFpjY^Fb%tROEZCah(c3XXe3T1T+e$_EV<{}I{0GY2d6f0R2 zj2N@jGNgWJ2=7C}b?|9mF>D7ee+^Wl^B_Vs81^DWPX83*TE{23;AlKfv`vW`i4 zWq*yu%H{E^Sci!M|H5%dP`~UY`o(S;T&cSex)yp!K~@7o$Uy)KVc2LnylVc$e$)Pt zf~*FFkW+yW#|*p4vgAhc!S-|}BZ5Q5WZ^}N--l~#hwbV@@%+@-o)x0a=rEQwFHyQg zA>!aW-sG|Uu&UsX@_!4h=mH3XkSHdD!SKLnGMFDZDMQb%sqRgx``1**Ce`s9>$J%* U@PNYfH!s`#hQomP26g@a4Rf9c`~Uy| literal 0 HcmV?d00001 diff --git a/dashboard/components/charts.py b/dashboard/components/charts.py new file mode 100644 index 0000000..49e505b --- /dev/null +++ b/dashboard/components/charts.py @@ -0,0 +1,371 @@ +""" +Chart components for Kafka Dashboard +Contains all chart generation functions using Plotly +""" + +import plotly.graph_objects as go +import plotly.express as px +from plotly.subplots import make_subplots +import pandas as pd +from typing import Dict, List, Any, Optional + + +class ChartBuilder: + """Builder class for creating Plotly charts""" + + @staticmethod + def create_health_score_gauge(health_score: float, title: str = "Cluster Health Score") -> go.Figure: + """Create a health score gauge chart""" + + # Determine color based on score + if health_score >= 90: + color = "#28a745" # Green + elif health_score >= 70: + color = "#ffc107" # Yellow + else: + color = "#dc3545" # Red + + fig = go.Figure(go.Indicator( + mode="gauge+number+delta", + value=health_score, + domain={'x': [0, 1], 'y': [0, 1]}, + title={'text': title, 'font': {'size': 20}}, + delta={'reference': 90, 'relative': True, 'valueformat': '.1%'}, + gauge={ + 'axis': {'range': [None, 100], 'tickwidth': 1, 'tickcolor': "darkblue"}, + 'bar': {'color': color}, + 'bgcolor': "white", + 'borderwidth': 2, + 'bordercolor': "gray", + 'steps': [ + {'range': [0, 50], 'color': '#f8d7da'}, + {'range': [50, 70], 'color': '#fff3cd'}, + {'range': [70, 90], 'color': '#d1ecf1'}, + {'range': [90, 100], 'color': '#d4edda'} + ], + 'threshold': { + 'line': {'color': "red", 'width': 4}, + 'thickness': 0.75, + 'value': 90 + } + } + )) + + fig.update_layout( + height=350, + font={'color': "darkblue", 'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + plot_bgcolor='rgba(0,0,0,0)' + ) + + return fig + + @staticmethod + def create_health_checks_summary(health_data: Dict[str, int]) -> go.Figure: + """Create health checks summary bar chart""" + + categories = ['Passed', 'Failed', 'Warnings'] + values = [ + health_data.get('passedChecks', 0), + health_data.get('failedChecks', 0), + health_data.get('warnings', 0) + ] + colors = ['#28a745', '#dc3545', '#ffc107'] + + fig = go.Figure() + + for i, (category, value, color) in enumerate(zip(categories, values, colors)): + fig.add_trace(go.Bar( + x=[category], + y=[value], + name=category, + marker_color=color, + text=[value], + textposition='auto', + hovertemplate=f'{category}
Count: %{{y}}' + )) + + fig.update_layout( + title="Health Checks Summary", + xaxis_title="Status", + yaxis_title="Count", + height=350, + showlegend=False, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + plot_bgcolor='rgba(0,0,0,0)', + xaxis={'gridcolor': '#e0e0e0'}, + yaxis={'gridcolor': '#e0e0e0'} + ) + + return fig + + @staticmethod + def create_topics_distribution(topics_data: Dict[str, int]) -> go.Figure: + """Create topics distribution pie chart""" + + labels = ['User Topics', 'Internal Topics'] + values = [ + topics_data.get('user_topics', 0), + topics_data.get('internal_topics', 0) + ] + colors = ['#007bff', '#6c757d'] + + fig = go.Figure(data=[ + go.Pie( + labels=labels, + values=values, + hole=0.4, + marker_colors=colors, + textinfo='label+percent+value', + hovertemplate='%{label}
Count: %{value}
Percentage: %{percent}' + ) + ]) + + fig.update_layout( + title="Topics Distribution", + height=350, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + annotations=[ + dict(text=f'Total
{sum(values)}', x=0.5, y=0.5, font_size=20, showarrow=False) + ] + ) + + return fig + + @staticmethod + def create_partitions_per_topic(topics: List[Dict[str, Any]]) -> go.Figure: + """Create partitions per topic bar chart""" + + # Filter out internal topics and prepare data + user_topics = [t for t in topics if not t.get('isInternal', False)] + + if not user_topics: + # Create empty chart + fig = go.Figure() + fig.add_annotation( + text="No user topics found", + xref="paper", yref="paper", + x=0.5, y=0.5, showarrow=False, + font={'size': 16, 'color': '#6c757d'} + ) + else: + topic_names = [t['name'] for t in user_topics] + partition_counts = [t.get('partitions', 0) for t in user_topics] + + fig = go.Figure(data=[ + go.Bar( + x=topic_names, + y=partition_counts, + marker_color='lightblue', + text=partition_counts, + textposition='auto', + hovertemplate='%{x}
Partitions: %{y}' + ) + ]) + + fig.update_layout( + title="Partitions per Topic", + xaxis_title="Topics", + yaxis_title="Partition Count", + height=350, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + plot_bgcolor='rgba(0,0,0,0)', + xaxis={'tickangle': -45, 'gridcolor': '#e0e0e0'}, + yaxis={'gridcolor': '#e0e0e0'} + ) + + return fig + + @staticmethod + def create_consumer_groups_chart(consumer_groups: List[Dict[str, Any]]) -> go.Figure: + """Create consumer groups status chart""" + + if not consumer_groups: + fig = go.Figure() + fig.add_annotation( + text="No consumer groups found", + xref="paper", yref="paper", + x=0.5, y=0.5, showarrow=False, + font={'size': 16, 'color': '#6c757d'} + ) + fig.update_layout(height=350, title="Consumer Groups Status") + return fig + + # Count groups by status + active_count = sum(1 for cg in consumer_groups if cg.get('members', 0) > 0) + inactive_count = len(consumer_groups) - active_count + + labels = ['Active', 'Inactive'] + values = [active_count, inactive_count] + colors = ['#28a745', '#ffc107'] + + fig = go.Figure(data=[ + go.Pie( + labels=labels, + values=values, + marker_colors=colors, + textinfo='label+percent+value', + hovertemplate='%{label}
Count: %{value}
Percentage: %{percent}' + ) + ]) + + fig.update_layout( + title="Consumer Groups Status", + height=350, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)' + ) + + return fig + + @staticmethod + def create_replication_factor_chart(topics: List[Dict[str, Any]]) -> go.Figure: + """Create replication factor distribution chart""" + + if not topics: + fig = go.Figure() + fig.add_annotation( + text="No topics found", + xref="paper", yref="paper", + x=0.5, y=0.5, showarrow=False, + font={'size': 16, 'color': '#6c757d'} + ) + fig.update_layout(height=350, title="Replication Factor Distribution") + return fig + + # Count topics by replication factor + rf_counts = {} + for topic in topics: + rf = topic.get('replicationFactor', 1) + rf_counts[rf] = rf_counts.get(rf, 0) + 1 + + rfs = list(rf_counts.keys()) + counts = list(rf_counts.values()) + + fig = go.Figure(data=[ + go.Bar( + x=[f"RF={rf}" for rf in rfs], + y=counts, + marker_color='lightcoral', + text=counts, + textposition='auto', + hovertemplate='%{x}
Topics: %{y}' + ) + ]) + + fig.update_layout( + title="Replication Factor Distribution", + xaxis_title="Replication Factor", + yaxis_title="Number of Topics", + height=350, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + plot_bgcolor='rgba(0,0,0,0)', + xaxis={'gridcolor': '#e0e0e0'}, + yaxis={'gridcolor': '#e0e0e0'} + ) + + return fig + + @staticmethod + def create_health_score_trend(trend_df: pd.DataFrame) -> go.Figure: + """Create health score trend line chart""" + + if trend_df.empty: + fig = go.Figure() + fig.add_annotation( + text="No historical data available", + xref="paper", yref="paper", + x=0.5, y=0.5, showarrow=False, + font={'size': 16, 'color': '#6c757d'} + ) + fig.update_layout(height=350, title="Health Score Trend") + return fig + + fig = go.Figure() + + fig.add_trace(go.Scatter( + x=trend_df['timestamp'], + y=trend_df['health_score'], + mode='lines+markers', + name='Health Score', + line=dict(color='#007bff', width=3), + marker=dict(size=8), + hovertemplate='Health Score
Date: %{x}
Score: %{y:.1f}%' + )) + + # Add threshold line + fig.add_hline( + y=90, + line_dash="dash", + line_color="red", + annotation_text="Target: 90%", + annotation_position="bottom right" + ) + + fig.update_layout( + title="Health Score Trend", + xaxis_title="Time", + yaxis_title="Health Score (%)", + height=350, + font={'family': "Inter, Arial"}, + paper_bgcolor='rgba(0,0,0,0)', + plot_bgcolor='rgba(0,0,0,0)', + xaxis={'gridcolor': '#e0e0e0'}, + yaxis={'gridcolor': '#e0e0e0', 'range': [0, 100]} + ) + + return fig + + +class MetricsCards: + """Helper class for creating metric cards data""" + + @staticmethod + def create_cluster_metrics(cluster_info: Dict[str, Any], topics_summary: Dict[str, Any], + consumer_summary: Dict[str, Any]) -> List[Dict[str, Any]]: + """Create metrics cards data""" + + metrics = [ + { + 'title': 'Total Brokers', + 'value': cluster_info.get('total_brokers', 0), + 'icon': 'đŸ–Ĩī¸', + 'color': 'primary' + }, + { + 'title': 'Total Topics', + 'value': topics_summary.get('total_topics', 0), + 'icon': '📋', + 'color': 'info' + }, + { + 'title': 'Total Partitions', + 'value': topics_summary.get('total_partitions', 0), + 'icon': '📊', + 'color': 'success' + }, + { + 'title': 'Consumer Groups', + 'value': consumer_summary.get('total_groups', 0), + 'icon': 'đŸ‘Ĩ', + 'color': 'warning' + }, + { + 'title': 'Active Groups', + 'value': consumer_summary.get('active_groups', 0), + 'icon': '✅', + 'color': 'success' + }, + { + 'title': 'Avg Partitions/Topic', + 'value': f"{topics_summary.get('avg_partitions_per_topic', 0):.1f}", + 'icon': '⚖ī¸', + 'color': 'secondary' + } + ] + + return metrics diff --git a/dashboard/components/layout.py b/dashboard/components/layout.py new file mode 100644 index 0000000..c37746f --- /dev/null +++ b/dashboard/components/layout.py @@ -0,0 +1,257 @@ +""" +Layout components for Kafka Dashboard +Contains reusable UI components and layouts +""" + +import dash_bootstrap_components as dbc +from dash import html, dcc, dash_table +from typing import List, Dict, Any + + +class LayoutComponents: + """Collection of reusable layout components""" + + @staticmethod + def create_header() -> dbc.Container: + """Create dashboard header""" + return dbc.Container([ + dbc.Row([ + dbc.Col([ + html.Div([ + html.H1([ + "🚀 ", + html.Span("Kafka Cluster Dashboard", className="text-primary"), + ], className="text-center mb-3"), + html.P( + "Real-time monitoring and analysis of your Kafka cluster health", + className="text-center text-muted lead" + ), + html.Hr(className="my-4") + ]) + ], width=12) + ]) + ], fluid=True, className="mb-4") + + @staticmethod + def create_metrics_cards(metrics: List[Dict[str, Any]]) -> dbc.Row: + """Create metrics cards row""" + cards = [] + + for metric in metrics: + card = dbc.Col([ + dbc.Card([ + dbc.CardBody([ + html.Div([ + html.Div([ + html.H2(metric['icon'], className="text-center mb-0"), + ], className="col-auto"), + html.Div([ + html.H4(str(metric['value']), className="mb-0 text-primary"), + html.P(metric['title'], className="text-muted small mb-0") + ], className="col") + ], className="row align-items-center") + ]) + ], className=f"border-left-{metric['color']} shadow-sm h-100") + ], width=12, lg=2, className="mb-3") + cards.append(card) + + return dbc.Row(cards) + + @staticmethod + def create_chart_card(chart_id: str, title: str, description: str = "") -> dbc.Card: + """Create a chart card wrapper""" + return dbc.Card([ + dbc.CardHeader([ + html.H5(title, className="mb-0"), + html.Small(description, className="text-muted") if description else None + ]), + dbc.CardBody([ + dcc.Graph(id=chart_id, config={'displayModeBar': False}) + ]) + ], className="shadow-sm h-100") + + @staticmethod + def create_health_details_table(table_id: str) -> dbc.Card: + """Create health details table card""" + return dbc.Card([ + dbc.CardHeader([ + html.H5("📋 Detailed Health Checks", className="mb-0"), + html.Small("Comprehensive analysis of cluster health", className="text-muted") + ]), + dbc.CardBody([ + html.Div(id=table_id) + ]) + ], className="shadow-sm") + + @staticmethod + def create_cluster_info_card(info_id: str) -> dbc.Card: + """Create cluster information card""" + return dbc.Card([ + dbc.CardHeader([ + html.H5("đŸĸ Cluster Information", className="mb-0"), + html.Small("General cluster metadata and configuration", className="text-muted") + ]), + dbc.CardBody([ + html.Div(id=info_id) + ]) + ], className="shadow-sm h-100") + + @staticmethod + def create_status_badge(status: str) -> html.Span: + """Create status badge based on health check status""" + badge_config = { + 'PASSED': {'color': 'success', 'icon': '✅'}, + 'FAILED': {'color': 'danger', 'icon': '❌'}, + 'WARNING': {'color': 'warning', 'icon': '⚠ī¸'}, + 'INFO': {'color': 'info', 'icon': 'ℹī¸'} + } + + config = badge_config.get(status, {'color': 'secondary', 'icon': '?'}) + + return dbc.Badge([ + config['icon'], " ", status + ], color=config['color'], className="me-2") + + @staticmethod + def create_data_table(data: List[Dict[str, Any]], table_id: str) -> dash_table.DataTable: + """Create a styled data table""" + if not data: + return html.Div([ + html.P("No data available", className="text-muted text-center p-4") + ]) + + columns = [ + {"name": "Status", "id": "status", "type": "text"}, + {"name": "Health Check", "id": "name", "type": "text"}, + {"name": "Message", "id": "message", "type": "text"}, + {"name": "Recommendation", "id": "recommendation", "type": "text"} + ] + + return dash_table.DataTable( + id=table_id, + data=data, + columns=columns, + style_cell={ + 'textAlign': 'left', + 'whiteSpace': 'normal', + 'height': 'auto', + 'maxWidth': '300px', + 'fontFamily': 'Inter, Arial', + 'fontSize': '14px', + 'padding': '12px' + }, + style_header={ + 'backgroundColor': '#f8f9fa', + 'fontWeight': 'bold', + 'border': '1px solid #dee2e6' + }, + style_data={ + 'border': '1px solid #dee2e6', + 'backgroundColor': 'white' + }, + style_data_conditional=[ + { + 'if': {'filter_query': '{status} contains ✅'}, + 'backgroundColor': '#d4edda', + 'color': 'black', + }, + { + 'if': {'filter_query': '{status} contains ❌'}, + 'backgroundColor': '#f8d7da', + 'color': 'black', + }, + { + 'if': {'filter_query': '{status} contains ⚠ī¸'}, + 'backgroundColor': '#fff3cd', + 'color': 'black', + }, + { + 'if': {'filter_query': '{status} contains ℹī¸'}, + 'backgroundColor': '#d1ecf1', + 'color': 'black', + } + ], + page_size=15, + sort_action="native", + filter_action="native", + style_table={'overflowX': 'auto'} + ) + + @staticmethod + def create_loading_spinner() -> dbc.Spinner: + """Create loading spinner""" + return dbc.Spinner([ + html.Div([ + html.H4("Loading Kafka data...", className="text-center text-muted"), + html.P("Please wait while we fetch the latest analysis", className="text-center text-muted") + ]) + ], size="lg", color="primary", type="border", fullscreen=True) + + @staticmethod + def create_no_data_message() -> html.Div: + """Create no data available message""" + return html.Div([ + dbc.Alert([ + html.H4("📊 No Data Available", className="alert-heading"), + html.P([ + "No Kafka analysis reports found. Please run the analyzer first:" + ]), + html.Hr(), + html.Pre([ + "cd /path/to/kafka-analyzer\n", + "npx superstream-kafka-analyzer --config config.json" + ], className="mb-0"), + html.P([ + html.Small("Then refresh this dashboard to view the results.") + ], className="mb-0 mt-2") + ], color="info", className="text-center") + ], className="my-5") + + @staticmethod + def create_refresh_controls() -> dbc.Row: + """Create refresh controls section""" + return dbc.Row([ + dbc.Col([ + dbc.ButtonGroup([ + dbc.Button( + "🔄 Refresh Now", + id="refresh-button", + color="primary", + size="sm", + className="me-2" + ), + dbc.Button( + "⚙ī¸ Settings", + id="settings-button", + color="outline-secondary", + size="sm" + ) + ]) + ], width="auto"), + dbc.Col([ + html.Div([ + html.Small("Last updated: ", className="text-muted"), + html.Small(id="last-updated", className="text-muted fw-bold") + ]) + ], width="auto", className="ms-auto") + ], className="mb-3 align-items-center") + + +class TabsLayout: + """Layout for tabbed interface""" + + @staticmethod + def create_tabs() -> dbc.Tabs: + """Create main dashboard tabs""" + return dbc.Tabs([ + dbc.Tab(label="📊 Overview", tab_id="overview"), + dbc.Tab(label="đŸĨ Health Checks", tab_id="health"), + dbc.Tab(label="📋 Topics", tab_id="topics"), + dbc.Tab(label="đŸ‘Ĩ Consumer Groups", tab_id="consumers"), + dbc.Tab(label="📈 Trends", tab_id="trends") + ], id="main-tabs", active_tab="overview") + + @staticmethod + def create_tab_content() -> html.Div: + """Create tab content container""" + return html.Div(id="tab-content", className="mt-4") diff --git a/dashboard/requirements.txt b/dashboard/requirements.txt new file mode 100644 index 0000000..7dc51d1 --- /dev/null +++ b/dashboard/requirements.txt @@ -0,0 +1,7 @@ +dash==2.16.1 +plotly==5.17.0 +pandas==2.1.4 +numpy>=1.26.0 +dash-bootstrap-components==1.5.0 +requests==2.31.0 +python-dateutil==2.8.2 diff --git a/dashboard/run_dashboard.py b/dashboard/run_dashboard.py new file mode 100755 index 0000000..d9fe52d --- /dev/null +++ b/dashboard/run_dashboard.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +Kafka Dashboard Launcher +Run this to start the interactive Kafka monitoring dashboard +""" + +import os +import sys +import argparse +import subprocess +from pathlib import Path + + +def check_dependencies(): + """Check if required Python packages are installed""" + required_packages = [ + 'dash', 'plotly', 'pandas', 'numpy', 'dash_bootstrap_components' + ] + + missing_packages = [] + + for package in required_packages: + try: + __import__(package) + except ImportError: + missing_packages.append(package) + + if missing_packages: + print("❌ Missing required packages:") + for package in missing_packages: + print(f" - {package}") + print("\n💡 Install missing packages with:") + print(" pip install -r requirements.txt") + return False + + return True + + +def find_data_directory(): + """Find the kafka-analysis directory""" + current_dir = Path.cwd() + + # Common locations to check + possible_paths = [ + current_dir / "kafka-analysis", + current_dir / ".." / "kafka-analysis", + current_dir / ".." / ".." / "kafka-analysis", + Path.home() / "kafka-analysis" + ] + + for path in possible_paths: + if path.exists() and path.is_dir(): + return str(path) + + return None + + +def check_analysis_files(data_dir): + """Check if analysis files exist in the data directory""" + if not data_dir or not os.path.exists(data_dir): + return False, [] + + json_files = [] + for file in os.listdir(data_dir): + if file.startswith('kafka-analysis-') and file.endswith('.json'): + json_files.append(file) + + return len(json_files) > 0, json_files + + +def install_dependencies(): + """Install required dependencies""" + print("đŸ“Ļ Installing required dependencies...") + try: + subprocess.check_call([ + sys.executable, "-m", "pip", "install", "-r", "requirements.txt" + ]) + print("✅ Dependencies installed successfully!") + return True + except subprocess.CalledProcessError: + print("❌ Failed to install dependencies") + return False + + +def main(): + parser = argparse.ArgumentParser( + description='Kafka Cluster Dashboard', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python run_dashboard.py # Run with auto-detected data directory + python run_dashboard.py --data-dir ./reports # Use custom data directory + python run_dashboard.py --port 8080 # Run on different port + python run_dashboard.py --install # Install dependencies first + """ + ) + + parser.add_argument('--data-dir', + help='Directory containing Kafka analysis reports') + parser.add_argument('--port', type=int, default=8050, + help='Port to run dashboard on (default: 8050)') + parser.add_argument('--host', default='127.0.0.1', + help='Host to bind to (default: 127.0.0.1)') + parser.add_argument('--debug', action='store_true', + help='Run in debug mode') + parser.add_argument('--install', action='store_true', + help='Install required dependencies first') + + args = parser.parse_args() + + # Change to dashboard directory + dashboard_dir = Path(__file__).parent + os.chdir(dashboard_dir) + + # Install dependencies if requested + if args.install: + if not install_dependencies(): + sys.exit(1) + + # Check dependencies + if not check_dependencies(): + print("\n💡 Run with --install flag to install dependencies automatically:") + print(" python run_dashboard.py --install") + sys.exit(1) + + # Determine data directory + if args.data_dir: + data_dir = args.data_dir + else: + data_dir = find_data_directory() + if not data_dir: + data_dir = "../kafka-analysis" # Default fallback + + # Check for analysis files + has_files, json_files = check_analysis_files(data_dir) + + print("🚀 Kafka Dashboard Launcher") + print("=" * 40) + print(f"📁 Data directory: {os.path.abspath(data_dir)}") + + if not has_files: + print("\n⚠ī¸ No analysis files found!") + print("💡 To generate analysis reports, run:") + print(" cd /path/to/kafka-analyzer") + print(" npx superstream-kafka-analyzer --config config.json") + print("\n📝 The dashboard will still start but show 'No data available'") + else: + print(f"✅ Found {len(json_files)} analysis file(s)") + print(f"📄 Latest: {max(json_files, key=lambda x: os.path.getmtime(os.path.join(data_dir, x)))}") + + print(f"\n🌐 Starting dashboard on http://{args.host}:{args.port}") + print("⚡ Dashboard will auto-refresh every 30 seconds") + print("🔄 Click 'Refresh Now' button to manually refresh") + print("\n📊 Dashboard Features:") + print(" â€ĸ Real-time cluster health monitoring") + print(" â€ĸ Interactive charts and metrics") + print(" â€ĸ Health checks analysis") + print(" â€ĸ Topics and consumer groups overview") + print(" â€ĸ Historical trend analysis") + + try: + # Import and run dashboard + from app import KafkaDashboard + + dashboard = KafkaDashboard(data_dir=data_dir) + dashboard.run(debug=args.debug, port=args.port, host=args.host) + + except KeyboardInterrupt: + print("\n👋 Dashboard stopped by user") + except Exception as e: + print(f"\n❌ Error starting dashboard: {e}") + print("💡 Try running with --debug flag for more details") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/dashboard/start_monitoring.py b/dashboard/start_monitoring.py new file mode 100755 index 0000000..2721f5c --- /dev/null +++ b/dashboard/start_monitoring.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +Kafka Analysis + Dashboard Integration +Runs the Kafka analyzer and then starts the dashboard +""" + +import os +import sys +import subprocess +import time +import argparse +from pathlib import Path + + +def run_kafka_analysis(config_file=None, bootstrap_servers=None): + """Run the Kafka analyzer to generate reports""" + print("🔍 Running Kafka Analysis...") + + # Change to parent directory (where the analyzer is) + original_dir = Path.cwd() + analyzer_dir = Path(__file__).parent.parent + os.chdir(analyzer_dir) + + try: + # Build command + cmd = ["node", "bin/index.js"] + + if config_file: + cmd.extend(["--config", config_file]) + elif bootstrap_servers: + cmd.extend(["--bootstrap-servers", bootstrap_servers]) + + # Run analyzer + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + print("✅ Kafka analysis completed successfully!") + return True + else: + print("❌ Kafka analysis failed:") + print(result.stderr) + return False + + except Exception as e: + print(f"❌ Error running analyzer: {e}") + return False + finally: + os.chdir(original_dir) + + +def start_dashboard(port=8050, host='127.0.0.1'): + """Start the dashboard""" + print("🚀 Starting Dashboard...") + + try: + # Change to dashboard directory + dashboard_dir = Path(__file__).parent + os.chdir(dashboard_dir) + + # Start dashboard + subprocess.run([ + sys.executable, "run_dashboard.py", + "--port", str(port), + "--host", host + ]) + + except KeyboardInterrupt: + print("\n👋 Dashboard stopped by user") + except Exception as e: + print(f"❌ Error starting dashboard: {e}") + + +def main(): + parser = argparse.ArgumentParser( + description='Run Kafka analysis and start dashboard', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python start_monitoring.py --config ../kraft-config.json # Use config file + python start_monitoring.py --servers localhost:29092 # Use servers directly + python start_monitoring.py --dashboard-only # Skip analysis, start dashboard only + """ + ) + + parser.add_argument('--config', + help='Path to Kafka analyzer config file') + parser.add_argument('--servers', + help='Kafka bootstrap servers (comma-separated)') + parser.add_argument('--port', type=int, default=8050, + help='Dashboard port (default: 8050)') + parser.add_argument('--host', default='127.0.0.1', + help='Dashboard host (default: 127.0.0.1)') + parser.add_argument('--dashboard-only', action='store_true', + help='Skip analysis, start dashboard only') + parser.add_argument('--analyze-only', action='store_true', + help='Run analysis only, skip dashboard') + + args = parser.parse_args() + + print("đŸŽ¯ Kafka Monitoring Setup") + print("=" * 40) + + # Step 1: Run analysis (unless dashboard-only) + if not args.dashboard_only: + if not args.config and not args.servers: + print("❌ Error: Must provide either --config or --servers") + print("💡 Examples:") + print(" python start_monitoring.py --config ../kraft-config.json") + print(" python start_monitoring.py --servers localhost:29092") + sys.exit(1) + + analysis_success = run_kafka_analysis( + config_file=args.config, + bootstrap_servers=args.servers + ) + + if not analysis_success: + print("❌ Analysis failed. Dashboard will show 'No data available'") + if not input("Continue with dashboard anyway? (y/N): ").lower().startswith('y'): + sys.exit(1) + + # Step 2: Start dashboard (unless analyze-only) + if not args.analyze_only: + print(f"\n🌐 Dashboard will be available at: http://{args.host}:{args.port}") + time.sleep(2) # Give user time to read + start_dashboard(port=args.port, host=args.host) + else: + print("✅ Analysis complete. Dashboard not started (--analyze-only flag)") + + +if __name__ == "__main__": + main() diff --git a/dashboard/utils/__pycache__/data_loader.cpython-312.pyc b/dashboard/utils/__pycache__/data_loader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d660636393b9f79ecfed93aabad652970783c96f GIT binary patch literal 11531 zcmcIqYj7Lab>0Pb@dhD~5&#l>fGG81~*PGpaiD$~xCFq9D4uv2B~AMroPbjG#) zr{~jI6K7B$ioD6n}$rJT@y49nKjfQ zieq1;INjSC?=33zO4@R)&T-F-aw&zIpWz-gbddTp|etTev2qvW+k9{Fl-2Hu08 zybZsP3kbtmB_T_VWi9-#lmnTeqI8tX0!4GipuwjneAX}`^Kx;;>jvV&A3)k%r;ISoxcN73;2TKK&7IqDqUNJ+Ko z=q5^VP3xq3wa)3Ix;LpGYSjDTSdO07y{wypb;93s^f^kA<26KDaQAzp)4XRqEJ}d4 z!+c08jlu(8Y$(`ZfX-W6+Q!6i=o&3+&+t<+EnL&ehLJ#!5BbM=r8MD}PRrJyUzB{~ zVJ`31f`=}gk<6e(*>K7)lDP=DFJ=U!XEfwb8X)v0-M%r5!U}m%8FjSGUe>y;ISV zcDg^+QO?Sl(MzK=1S$|%J_~{<(AkJzpbcI*xj|W zWAB{$@#wYDrM9kQTi2bceX;(J>>Dngi9edMZ^}@#wJFiL>~zmeU7Cv9zaPEGe5QrA zPfb)!-IYU^4=q)^`MuJX z|Ljuz3(5KyQuU|q>6!YnSnox5(o&PLQ0rZ>{?9gmP;$2~KXp;-DnFq#@RSz&;`)|E zXQFYjyzP#qeYwKY}ODmXn#O>3pE zhtfIw&`Ul4_Y@#9*t0xZGh7bE0j=3J(3(}TXj;cH`1iHYI*0vH=1-2+Ioaxr*FndZ|&Z1Ew=?g4-R{1G4AlNRXYG z*1xQO?L&_I=8+vNM4_~*B56h+XXEC>N@6AR0>nZ5@Gs0oPdY z0FiYf#05MjG++xZl+c8v8OcT@xXFc0NHznJb^Zx351in|mKGph1A%y93pTVOi#~gI z*@(#QL*pZ~VQ)K<4j>|;M;75d3dGwGzhPl~*};*zQUG7TunpxrnT7WnqoXvuS6(!J zrOrj6d;{13x`<{m|03-sHC4J2icuP&RALMPo)&X0HK@FdIlzq$;-p z?rvzEw=dTflCX248;apj`EU{4lrmm9qnhU5PF8&n(dM;$qeAJL~r> zudhbS79K=LyNpvS3T`X_EW9dj&Y-X^28O0^U2Pi$F91J6AJEdG!%*k3Tp( ziGGLYH17{er#<3uSl~VA|3JK*4XugSVo?}+PV>WO#8GupvRMjC{-Dw-VDKVaCj6qv zb4t_495kWZ2g>hdm3@SpM7Ap8e8WVZY$0OQnpF{zt%?}`kiDdAq;JU&!%jquO5T4Q3eHhBeOvbi&(aN{%Yf z8i9Vz5oIoEUaMrl7NdTAbs1uz3?0N@_>APjBoIneV-(e*P3xmtp)9JGa&c@v)9j$VQKvTu}q3S68WEZidoxb$buCS%^D$yV=><3nJ-Gz{8~%qZ07swM0aKt~ ztK>A<#i)hF(2~VqWFwDJVow;7i+JCU5k-{+5spBya4`hD{U5mulPIeZh_VdL&nCJT zI)AqPz3n&LxBaQz#}<#DNbNrHBY(2_+0R$d6L_VniTMzHG$? z@g&h0wPBrtiupGzyZmQI!QW8dxNkxoEth9s-noEu+DDc_^fE#h++5be3zfAHY|E?= zo(yqbV;+bFFb|Z_hom3LAd=%q5Hsbn;`@;dm+IDJ{hD$F+hWa$G_0*%FMxP)2?&8? zu+Xt1cgr0!n=fsi-4#ETD)+|pY5Rs5*Cp5N=J)}KI%B$wj%xJ&%F#dj^gR7j_BuNs zywUf|jwinN;-aHJW(Jbp*!rug{qvws z62fO7i600lxe-91Qpji?4o-@o)<=L%)+=K~*{HU1vi?NqOelN~GAF~~kR*hIL0*8A zy;aeDb$}^5z!%_zQ?O71=83%;qTm8rc2IVVV~NitTXNhuIbvYdPnRK>*$539oRNjy*Z;0OE$ zC(~6Mm#SKlRV|6#3&vE{fqZ?>LS?F|C+)1gXJk=}SflBFUjmH=(UB+=9;PEw&Tr&2 zVDU61tg*I0M{>&p6Z$NN? zC2${)H!M8zv&Y_h?1uOD)2ZE0Ej}}t+CBJIchY_QbKz-V2F=d;x{7Ss3Jyl#ET6rV z3^t&+L8tJ+Lwo~`TNZVFTRv1UN5Y7v=hJdxA^KJy{Rov>MMYeo}b@)pZp7W*` zotqL;()p;e8LDa*>)Ml59WmQSE;l@N!gj}fJmoqbvo2TF%!X4{ZSytBs;-z#0a{zC zx+7iFlCEn@JDMN11HeyIcEInTP`m>$i?(L$RD_a{^(ydN^GFF9tOFEv5?lxHj&It##E<@E#15RrOk1UQ)?X?z`I%cV(ChJ7fCaOftSh52~3iVtGs zLx>d+X$qLv$fnJegvmZ=5Ja53O?^r=5~Kq%t2_`o_(Yji2(RH4?_3Q96*iwDL5yr& ztM^R8|a?{}?WHPcYR z*3CD~iwiyT=dZVAD5%~ze!J^sfBP$p`t@^G2 zzAE5Z{F z0rLMda6ne26y`&Yu5>$i=qiGj9G5#UR=FVOLXYKg75PczZO}9e-Z(z*stbHD_5rhC z)|dsH6RY?CBPHm+wgf|70_0o@ZQZ{zRKa%zs5q;xyeA4aSvQ6Vv+5FC3nXA7Zq*gl zDKV6qk;3{O-~nkBt5~55>6WY)jrRj9Y>Ide@setIl?gjU z_)P#Y$im}TLX{JC1CcEw#EcWOEJ8~3uSsKJI#bw>%-z^-RhYBZ&}&j~7I%^o_#D6z z+EZCG0mtRI!4C>}q|Wl=6VeoBXMM1z$!Acq84p5yXZ^uRUQ~~*&A8*UU%JR9@=!Ry zkKe7}_f%wh!t3M?+{pds){fPgRv(cOX?xJNF3tZFxPgctU0t9)ax|wLn_}j>8=Nsy z+EImv)}K%&qb+tY?W(<97~oOI8{#9Y)pGv)1Hl2O>?G1*TN$yPuG&?K+<#I#$I&P zJ;&4ao32*PRV4-%4y5Y$E!FoV>w9htU^=J%*@yLk(XPuHZ6iz-4y()7y>arpCudty z*5;TdZL_~|{;$q2*LtpmE{77&%$u+IQnhH;9sjFoCH1hP{7_X8OoK$PMv z03!l+s_iS7NmL82b*Zt1KvkNf5Qs{16a!7c5$J2L1JLB4cMQ{VtFD3%nAj*=>Jujb zn!JQkPz4_uRf1F%Kp;?1fQB7IYq#nu02;U-0h%fV8YP+LSp_l5R>b710xrE!${xQQ zC;4dtGO`IXib{iQBRP~~xpX1qNDyv>XONIg-w7ze`DPwb5KM$OkX%G^3CZ`6d>@Ec zrwc*!VC}*9ZJPXS3*v5RFlmOD!{P$VZczx`5!3nLm(HZVkj_v6)132 z&x~Cfi&rKNq#W(BL&Z?A>8g3ooOp7fGv(QvuG@I!rOPiR_AJ;_b-U9|Tdwxc^)GgG z-{?#=^+8~Y+nWWS+z7Xa6Unc?y$8Hj;Q~CK@Es(tAo+77#Q$cn^ePhKm=k9`|57kc z5q^l|FOZ6#oVUxm3HTn@FDkl%n;DKirY7LSTqU3&k3r%K z@&q0nM}h=0$KrW^Vc4U@EL}nz_yFN3aVLn*XL+ fQ&j73DD!8!=QVWQJqqb(``J#q{1XZYnfCtyS&e6{ literal 0 HcmV?d00001 diff --git a/dashboard/utils/data_loader.py b/dashboard/utils/data_loader.py new file mode 100644 index 0000000..3911302 --- /dev/null +++ b/dashboard/utils/data_loader.py @@ -0,0 +1,223 @@ +""" +Data loader utilities for Kafka Dashboard +Handles loading and processing Kafka analysis reports +""" + +import json +import os +import pandas as pd +from datetime import datetime +from typing import Dict, List, Optional, Any +import glob + + +class KafkaDataLoader: + """Handles loading and processing Kafka analysis data""" + + def __init__(self, data_dir: str = "../kafka-analysis"): + self.data_dir = data_dir + + def get_latest_report(self) -> Optional[Dict[str, Any]]: + """Load the most recent Kafka analysis report""" + try: + if not os.path.exists(self.data_dir): + return None + + # Find all JSON analysis files + pattern = os.path.join(self.data_dir, "kafka-analysis-*.json") + json_files = glob.glob(pattern) + + if not json_files: + return None + + # Get the most recent file by modification time + latest_file = max(json_files, key=os.path.getmtime) + + with open(latest_file, 'r') as f: + data = json.load(f) + + # Add file metadata + data['_metadata'] = { + 'filename': os.path.basename(latest_file), + 'filepath': latest_file, + 'last_modified': datetime.fromtimestamp(os.path.getmtime(latest_file)).isoformat() + } + + return data + + except Exception as e: + print(f"Error loading report: {e}") + return None + + def get_all_reports(self) -> List[Dict[str, Any]]: + """Load all available Kafka analysis reports""" + try: + if not os.path.exists(self.data_dir): + return [] + + pattern = os.path.join(self.data_dir, "kafka-analysis-*.json") + json_files = glob.glob(pattern) + + reports = [] + for file_path in sorted(json_files, key=os.path.getmtime): + try: + with open(file_path, 'r') as f: + data = json.load(f) + + # Add metadata + data['_metadata'] = { + 'filename': os.path.basename(file_path), + 'filepath': file_path, + 'last_modified': datetime.fromtimestamp(os.path.getmtime(file_path)).isoformat() + } + + reports.append(data) + except Exception as e: + print(f"Error loading {file_path}: {e}") + continue + + return reports + + except Exception as e: + print(f"Error loading reports: {e}") + return [] + + def get_health_score(self, data: Dict[str, Any]) -> float: + """Calculate health score from analysis data""" + if not data or 'healthChecks' not in data: + return 0.0 + + health_checks = data['healthChecks'] + total_checks = health_checks.get('totalChecks', 0) + passed_checks = health_checks.get('passedChecks', 0) + + return (passed_checks / total_checks * 100) if total_checks > 0 else 0.0 + + def get_topics_summary(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Extract topics summary from analysis data""" + if not data: + return {} + + summary = data.get('summary', {}) + topics = data.get('topics', []) + + # Calculate additional metrics + user_topics = [t for t in topics if not t.get('isInternal', False)] + internal_topics = [t for t in topics if t.get('isInternal', False)] + + total_partitions = sum(t.get('partitions', 0) for t in topics) + avg_partitions = total_partitions / len(topics) if topics else 0 + + return { + 'total_topics': len(topics), + 'user_topics': len(user_topics), + 'internal_topics': len(internal_topics), + 'total_partitions': total_partitions, + 'avg_partitions_per_topic': round(avg_partitions, 2), + 'topics_with_errors': sum(1 for t in topics if t.get('errorCode', 0) != 0) + } + + def get_broker_info(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Extract broker information from analysis data""" + if not data or 'clusterInfo' not in data: + return {} + + cluster_info = data['clusterInfo'] + brokers = cluster_info.get('brokers', []) + + return { + 'total_brokers': len(brokers), + 'cluster_id': cluster_info.get('clusterId', 'Unknown'), + 'controller': cluster_info.get('controller', 'Unknown'), + 'brokers': brokers + } + + def get_consumer_groups_summary(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Extract consumer groups summary""" + if not data or 'consumerGroups' not in data: + return {} + + consumer_groups = data['consumerGroups'] + + active_groups = sum(1 for cg in consumer_groups if cg.get('members', 0) > 0) + inactive_groups = len(consumer_groups) - active_groups + + return { + 'total_groups': len(consumer_groups), + 'active_groups': active_groups, + 'inactive_groups': inactive_groups, + 'groups': consumer_groups + } + + def extract_health_checks_details(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: + """Extract detailed health checks information""" + if not data or 'healthChecks' not in data: + return [] + + health_checks = data['healthChecks'] + checks = health_checks.get('checks', []) + + detailed_checks = [] + for i, check in enumerate(checks): + # Map the check details (this would need to be enhanced based on actual data structure) + detailed_checks.append({ + 'id': i, + 'name': f"Health Check {i+1}", + 'status': 'PASSED', # This would come from actual data + 'message': check.get('description', 'No description available'), + 'recommendation': check.get('recommendation', 'No recommendation') + }) + + return detailed_checks + + +class HistoricalDataProcessor: + """Process historical data for trend analysis""" + + def __init__(self, reports: List[Dict[str, Any]]): + self.reports = reports + + def get_health_score_trend(self) -> pd.DataFrame: + """Get health score trend over time""" + data_loader = KafkaDataLoader() + + trend_data = [] + for report in self.reports: + timestamp = report.get('timestamp', report.get('_metadata', {}).get('last_modified')) + health_score = data_loader.get_health_score(report) + + trend_data.append({ + 'timestamp': timestamp, + 'health_score': health_score, + 'total_checks': report.get('healthChecks', {}).get('totalChecks', 0), + 'passed_checks': report.get('healthChecks', {}).get('passedChecks', 0), + 'failed_checks': report.get('healthChecks', {}).get('failedChecks', 0) + }) + + df = pd.DataFrame(trend_data) + if not df.empty: + df['timestamp'] = pd.to_datetime(df['timestamp']) + df = df.sort_values('timestamp') + + return df + + def get_topics_trend(self) -> pd.DataFrame: + """Get topics trend over time""" + trend_data = [] + for report in self.reports: + timestamp = report.get('timestamp', report.get('_metadata', {}).get('last_modified')) + summary = report.get('summary', {}) + + trend_data.append({ + 'timestamp': timestamp, + 'total_topics': summary.get('totalTopics', 0), + 'user_topics': summary.get('userTopics', 0), + 'total_partitions': summary.get('totalPartitions', 0) + }) + + df = pd.DataFrame(trend_data) + if not df.empty: + df['timestamp'] = pd.to_datetime(df['timestamp']) + df = df.sort_values('timestamp') + + return df diff --git a/kafka-analysis/kafka-analysis-1752105684783.json b/kafka-analysis/kafka-analysis-1752105684783.json new file mode 100644 index 0000000..d39f551 --- /dev/null +++ b/kafka-analysis/kafka-analysis-1752105684783.json @@ -0,0 +1,698 @@ +{ + "clusterInfo": { + "clusterId": "MkU3OEVBNTcwNTJENDM2Qk", + "controller": 1, + "brokers": [ + { + "nodeId": 1, + "host": "localhost", + "port": 29092 + } + ], + "topics": 2 + }, + "topics": [ + { + "name": "test_topic", + "partitions": 1, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": false, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + }, + { + "name": "__consumer_offsets", + "partitions": 50, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": true, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 7, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 1, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 42, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 13, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 25, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 36, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 31, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 37, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 43, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 18, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 24, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 8, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 14, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 30, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 29, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 32, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 26, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 38, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 17, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 41, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 20, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 47, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 49, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 15, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 23, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 44, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 12, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 9, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 3, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 6, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 35, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 10, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 45, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 16, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 4, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 33, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 28, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 39, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 40, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 5, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 34, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 21, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 46, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 11, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 27, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 2, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 22, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 19, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 48, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + } + ], + "consumerGroups": [ + { + "groupId": "perf-test-group", + "protocolType": "unknown", + "members": 0, + "state": "error" + } + ], + "summary": { + "totalTopics": 2, + "totalPartitions": 51, + "internalTopics": 1, + "userTopics": 1, + "topicsWithErrors": 0, + "consumerGroups": 1 + }, + "timestamp": "2025-07-10T00:01:24.675Z", + "healthChecks": { + "vendor": "apache-kafka", + "totalChecks": 14, + "passedChecks": 7, + "failedChecks": 1, + "warnings": 5, + "checks": [ + { + "id": "replication-factor", + "name": "Replication Factor vs Broker Count", + "status": "pass", + "message": "All topics have appropriate replication factor (≤ 1 brokers)", + "recommendation": null, + "description": "Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count." + }, + { + "id": "partition-distribution", + "name": "Topic Partition Distribution", + "status": "pass", + "message": "Good partition distribution: avg=1.0, min=1, max=1", + "recommendation": null, + "description": "Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions." + }, + { + "id": "consumer-groups", + "name": "Consumer Group Health", + "status": "warning", + "message": "1 consumer group(s) have no active members: perf-test-group", + "recommendation": "Consider cleaning up unused consumer groups", + "description": "Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members." + }, + { + "id": "internal-topics", + "name": "Internal Topics Health", + "status": "pass", + "message": "All 1 internal topics are healthy", + "recommendation": null, + "description": "Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions." + }, + { + "id": "under-replicated-partitions", + "name": "Under-Replicated Partitions", + "status": "pass", + "message": "All topics have the expected number of in-sync replicas", + "recommendation": null, + "description": "" + }, + { + "id": "min-insync-replicas", + "name": "Min In-Sync Replicas Configuration", + "status": "pass", + "message": "All topics have appropriate min.insync.replicas configuration", + "recommendation": null, + "description": "" + }, + { + "id": "rack-awareness", + "name": "Rack Awareness", + "status": "warning", + "message": "Rack awareness is not configured - no brokers have rack information", + "recommendation": "Consider enabling rack awareness for better availability and fault tolerance", + "description": "Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured." + }, + { + "id": "replica-distribution", + "name": "Replica Distribution", + "status": "pass", + "message": "Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)", + "recommendation": null, + "description": "Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues." + }, + { + "id": "metrics-enabled", + "name": "Metrics Configuration", + "status": "warning", + "message": "No JMX metrics configuration detected on any brokers", + "recommendation": "Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis", + "description": "Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured." + }, + { + "id": "logging-configuration", + "name": "Generic Kafka Logging Configuration", + "status": "info", + "message": "Generic Kafka logging configuration check", + "recommendation": "Verify log4j configuration and log directory permissions in server.properties", + "description": "Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured." + }, + { + "id": "authentication-configuration", + "name": "Generic Kafka Authentication Configuration", + "status": "fail", + "message": "Unauthenticated access is enabled - this is a security risk", + "recommendation": "Enable SASL or SSL authentication in server.properties for better security", + "description": "Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk)." + }, + { + "id": "quotas-configuration", + "name": "Generic Kafka Quotas Configuration", + "status": "warning", + "message": "No quota configuration detected in Kafka cluster", + "recommendation": "Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management", + "description": "Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available." + }, + { + "id": "payload-compression", + "name": "Payload Compression", + "status": "warning", + "message": "No compression detected on any of the 1 user topics (0%)", + "recommendation": "Enable compression on topics to reduce storage usage and improve network performance", + "description": "Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze." + }, + { + "id": "infinite-retention-policy", + "name": "Infinite Retention Policy", + "status": "pass", + "message": "No topics have infinite retention policy enabled", + "recommendation": null, + "description": "Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy." + } + ] + } +} \ No newline at end of file diff --git a/kafka-analysis/kafka-analysis-1752105715380.json b/kafka-analysis/kafka-analysis-1752105715380.json new file mode 100644 index 0000000..5e6dfd2 --- /dev/null +++ b/kafka-analysis/kafka-analysis-1752105715380.json @@ -0,0 +1,698 @@ +{ + "clusterInfo": { + "clusterId": "MkU3OEVBNTcwNTJENDM2Qk", + "controller": 1, + "brokers": [ + { + "nodeId": 1, + "host": "localhost", + "port": 29092 + } + ], + "topics": 2 + }, + "topics": [ + { + "name": "test_topic", + "partitions": 1, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": false, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + }, + { + "name": "__consumer_offsets", + "partitions": 50, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": true, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 7, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 1, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 42, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 13, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 25, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 36, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 31, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 37, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 43, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 18, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 24, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 8, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 14, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 30, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 29, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 32, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 26, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 38, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 17, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 41, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 20, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 47, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 49, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 15, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 23, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 44, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 12, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 9, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 3, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 6, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 35, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 10, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 45, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 16, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 4, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 33, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 28, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 39, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 40, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 5, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 34, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 21, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 46, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 11, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 27, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 2, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 22, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 19, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 48, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + } + ], + "consumerGroups": [ + { + "groupId": "perf-test-group", + "protocolType": "unknown", + "members": 0, + "state": "error" + } + ], + "summary": { + "totalTopics": 2, + "totalPartitions": 51, + "internalTopics": 1, + "userTopics": 1, + "topicsWithErrors": 0, + "consumerGroups": 1 + }, + "timestamp": "2025-07-10T00:01:55.307Z", + "healthChecks": { + "vendor": "apache-kafka", + "totalChecks": 14, + "passedChecks": 7, + "failedChecks": 1, + "warnings": 5, + "checks": [ + { + "id": "replication-factor", + "name": "Replication Factor vs Broker Count", + "status": "pass", + "message": "All topics have appropriate replication factor (≤ 1 brokers)", + "recommendation": null, + "description": "Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count." + }, + { + "id": "partition-distribution", + "name": "Topic Partition Distribution", + "status": "pass", + "message": "Good partition distribution: avg=1.0, min=1, max=1", + "recommendation": null, + "description": "Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions." + }, + { + "id": "consumer-groups", + "name": "Consumer Group Health", + "status": "warning", + "message": "1 consumer group(s) have no active members: perf-test-group", + "recommendation": "Consider cleaning up unused consumer groups", + "description": "Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members." + }, + { + "id": "internal-topics", + "name": "Internal Topics Health", + "status": "pass", + "message": "All 1 internal topics are healthy", + "recommendation": null, + "description": "Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions." + }, + { + "id": "under-replicated-partitions", + "name": "Under-Replicated Partitions", + "status": "pass", + "message": "All topics have the expected number of in-sync replicas", + "recommendation": null, + "description": "" + }, + { + "id": "min-insync-replicas", + "name": "Min In-Sync Replicas Configuration", + "status": "pass", + "message": "All topics have appropriate min.insync.replicas configuration", + "recommendation": null, + "description": "" + }, + { + "id": "rack-awareness", + "name": "Rack Awareness", + "status": "warning", + "message": "Rack awareness is not configured - no brokers have rack information", + "recommendation": "Consider enabling rack awareness for better availability and fault tolerance", + "description": "Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured." + }, + { + "id": "replica-distribution", + "name": "Replica Distribution", + "status": "pass", + "message": "Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)", + "recommendation": null, + "description": "Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues." + }, + { + "id": "metrics-enabled", + "name": "Metrics Configuration", + "status": "warning", + "message": "No JMX metrics configuration detected on any brokers", + "recommendation": "Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis", + "description": "Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured." + }, + { + "id": "logging-configuration", + "name": "Generic Kafka Logging Configuration", + "status": "info", + "message": "Generic Kafka logging configuration check", + "recommendation": "Verify log4j configuration and log directory permissions in server.properties", + "description": "Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured." + }, + { + "id": "authentication-configuration", + "name": "Generic Kafka Authentication Configuration", + "status": "fail", + "message": "Unauthenticated access is enabled - this is a security risk", + "recommendation": "Enable SASL or SSL authentication in server.properties for better security", + "description": "Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk)." + }, + { + "id": "quotas-configuration", + "name": "Generic Kafka Quotas Configuration", + "status": "warning", + "message": "No quota configuration detected in Kafka cluster", + "recommendation": "Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management", + "description": "Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available." + }, + { + "id": "payload-compression", + "name": "Payload Compression", + "status": "warning", + "message": "No compression detected on any of the 1 user topics (0%)", + "recommendation": "Enable compression on topics to reduce storage usage and improve network performance", + "description": "Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze." + }, + { + "id": "infinite-retention-policy", + "name": "Infinite Retention Policy", + "status": "pass", + "message": "No topics have infinite retention policy enabled", + "recommendation": null, + "description": "Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy." + } + ] + } +} \ No newline at end of file diff --git a/kafka-analysis/kafka-analysis-1752105730445.json b/kafka-analysis/kafka-analysis-1752105730445.json new file mode 100644 index 0000000..bb20ecc --- /dev/null +++ b/kafka-analysis/kafka-analysis-1752105730445.json @@ -0,0 +1,698 @@ +{ + "clusterInfo": { + "clusterId": "MkU3OEVBNTcwNTJENDM2Qk", + "controller": 1, + "brokers": [ + { + "nodeId": 1, + "host": "localhost", + "port": 29092 + } + ], + "topics": 2 + }, + "topics": [ + { + "name": "test_topic", + "partitions": 1, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": false, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + }, + { + "name": "__consumer_offsets", + "partitions": 50, + "replicationFactor": 1, + "config": { + "undefined": { + "isDefault": true, + "isSensitive": false + } + }, + "isInternal": true, + "errorCode": 0, + "errorMessage": "Topic has errors", + "vendor": "apache-kafka", + "partitionDetails": [ + { + "id": 7, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 1, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 42, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 13, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 25, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 36, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 31, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 37, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 43, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 18, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 24, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 8, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 14, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 30, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 29, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 32, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 26, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 38, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 0, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 17, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 41, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 20, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 47, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 49, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 15, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 23, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 44, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 12, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 9, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 3, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 6, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 35, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 10, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 45, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 16, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 4, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 33, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 28, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 39, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 40, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 5, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 34, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 21, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 46, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 11, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 27, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 2, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 22, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 19, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + }, + { + "id": 48, + "leader": 1, + "replicas": [ + 1 + ], + "isr": [ + 1 + ] + } + ] + } + ], + "consumerGroups": [ + { + "groupId": "perf-test-group", + "protocolType": "unknown", + "members": 0, + "state": "error" + } + ], + "summary": { + "totalTopics": 2, + "totalPartitions": 51, + "internalTopics": 1, + "userTopics": 1, + "topicsWithErrors": 0, + "consumerGroups": 1 + }, + "timestamp": "2025-07-10T00:02:10.369Z", + "healthChecks": { + "vendor": "apache-kafka", + "totalChecks": 14, + "passedChecks": 7, + "failedChecks": 1, + "warnings": 5, + "checks": [ + { + "id": "replication-factor", + "name": "Replication Factor vs Broker Count", + "status": "pass", + "message": "All topics have appropriate replication factor (≤ 1 brokers)", + "recommendation": null, + "description": "Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count." + }, + { + "id": "partition-distribution", + "name": "Topic Partition Distribution", + "status": "pass", + "message": "Good partition distribution: avg=1.0, min=1, max=1", + "recommendation": null, + "description": "Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions." + }, + { + "id": "consumer-groups", + "name": "Consumer Group Health", + "status": "warning", + "message": "1 consumer group(s) have no active members: perf-test-group", + "recommendation": "Consider cleaning up unused consumer groups", + "description": "Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members." + }, + { + "id": "internal-topics", + "name": "Internal Topics Health", + "status": "pass", + "message": "All 1 internal topics are healthy", + "recommendation": null, + "description": "Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions." + }, + { + "id": "under-replicated-partitions", + "name": "Under-Replicated Partitions", + "status": "pass", + "message": "All topics have the expected number of in-sync replicas", + "recommendation": null, + "description": "" + }, + { + "id": "min-insync-replicas", + "name": "Min In-Sync Replicas Configuration", + "status": "pass", + "message": "All topics have appropriate min.insync.replicas configuration", + "recommendation": null, + "description": "" + }, + { + "id": "rack-awareness", + "name": "Rack Awareness", + "status": "warning", + "message": "Rack awareness is not configured - no brokers have rack information", + "recommendation": "Consider enabling rack awareness for better availability and fault tolerance", + "description": "Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured." + }, + { + "id": "replica-distribution", + "name": "Replica Distribution", + "status": "pass", + "message": "Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)", + "recommendation": null, + "description": "Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues." + }, + { + "id": "metrics-enabled", + "name": "Metrics Configuration", + "status": "warning", + "message": "No JMX metrics configuration detected on any brokers", + "recommendation": "Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis", + "description": "Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured." + }, + { + "id": "logging-configuration", + "name": "Generic Kafka Logging Configuration", + "status": "info", + "message": "Generic Kafka logging configuration check", + "recommendation": "Verify log4j configuration and log directory permissions in server.properties", + "description": "Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured." + }, + { + "id": "authentication-configuration", + "name": "Generic Kafka Authentication Configuration", + "status": "fail", + "message": "Unauthenticated access is enabled - this is a security risk", + "recommendation": "Enable SASL or SSL authentication in server.properties for better security", + "description": "Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk)." + }, + { + "id": "quotas-configuration", + "name": "Generic Kafka Quotas Configuration", + "status": "warning", + "message": "No quota configuration detected in Kafka cluster", + "recommendation": "Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management", + "description": "Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available." + }, + { + "id": "payload-compression", + "name": "Payload Compression", + "status": "warning", + "message": "No compression detected on any of the 1 user topics (0%)", + "recommendation": "Enable compression on topics to reduce storage usage and improve network performance", + "description": "Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze." + }, + { + "id": "infinite-retention-policy", + "name": "Infinite Retention Policy", + "status": "pass", + "message": "No topics have infinite retention policy enabled", + "recommendation": null, + "description": "Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy." + } + ] + } +} \ No newline at end of file diff --git a/kafka-analysis/kafka-health-checks-1752105684783.csv b/kafka-analysis/kafka-health-checks-1752105684783.csv new file mode 100644 index 0000000..fac3ac0 --- /dev/null +++ b/kafka-analysis/kafka-health-checks-1752105684783.csv @@ -0,0 +1,16 @@ +"Health Check Results" +"Check Name","Status","Message","Description","Recommendation" +"Replication Factor vs Broker Count","pass","All topics have appropriate replication factor (≤ 1 brokers)","Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count.","" +"Topic Partition Distribution","pass","Good partition distribution: avg=1.0, min=1, max=1","Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions.","" +"Consumer Group Health","warning","1 consumer group(s) have no active members: perf-test-group","Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members.","Consider cleaning up unused consumer groups" +"Internal Topics Health","pass","All 1 internal topics are healthy","Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions.","" +"Under-Replicated Partitions","pass","All topics have the expected number of in-sync replicas","","" +"Min In-Sync Replicas Configuration","pass","All topics have appropriate min.insync.replicas configuration","","" +"Rack Awareness","warning","Rack awareness is not configured - no brokers have rack information","Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured.","Consider enabling rack awareness for better availability and fault tolerance" +"Replica Distribution","pass","Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)","Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues.","" +"Metrics Configuration","warning","No JMX metrics configuration detected on any brokers","Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured.","Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis" +"Generic Kafka Logging Configuration","info","Generic Kafka logging configuration check","Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured.","Verify log4j configuration and log directory permissions in server.properties" +"Generic Kafka Authentication Configuration","fail","Unauthenticated access is enabled - this is a security risk","Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk).","Enable SASL or SSL authentication in server.properties for better security" +"Generic Kafka Quotas Configuration","warning","No quota configuration detected in Kafka cluster","Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available.","Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management" +"Payload Compression","warning","No compression detected on any of the 1 user topics (0%)","Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze.","Enable compression on topics to reduce storage usage and improve network performance" +"Infinite Retention Policy","pass","No topics have infinite retention policy enabled","Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy.","" \ No newline at end of file diff --git a/kafka-analysis/kafka-health-checks-1752105715380.csv b/kafka-analysis/kafka-health-checks-1752105715380.csv new file mode 100644 index 0000000..fac3ac0 --- /dev/null +++ b/kafka-analysis/kafka-health-checks-1752105715380.csv @@ -0,0 +1,16 @@ +"Health Check Results" +"Check Name","Status","Message","Description","Recommendation" +"Replication Factor vs Broker Count","pass","All topics have appropriate replication factor (≤ 1 brokers)","Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count.","" +"Topic Partition Distribution","pass","Good partition distribution: avg=1.0, min=1, max=1","Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions.","" +"Consumer Group Health","warning","1 consumer group(s) have no active members: perf-test-group","Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members.","Consider cleaning up unused consumer groups" +"Internal Topics Health","pass","All 1 internal topics are healthy","Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions.","" +"Under-Replicated Partitions","pass","All topics have the expected number of in-sync replicas","","" +"Min In-Sync Replicas Configuration","pass","All topics have appropriate min.insync.replicas configuration","","" +"Rack Awareness","warning","Rack awareness is not configured - no brokers have rack information","Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured.","Consider enabling rack awareness for better availability and fault tolerance" +"Replica Distribution","pass","Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)","Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues.","" +"Metrics Configuration","warning","No JMX metrics configuration detected on any brokers","Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured.","Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis" +"Generic Kafka Logging Configuration","info","Generic Kafka logging configuration check","Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured.","Verify log4j configuration and log directory permissions in server.properties" +"Generic Kafka Authentication Configuration","fail","Unauthenticated access is enabled - this is a security risk","Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk).","Enable SASL or SSL authentication in server.properties for better security" +"Generic Kafka Quotas Configuration","warning","No quota configuration detected in Kafka cluster","Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available.","Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management" +"Payload Compression","warning","No compression detected on any of the 1 user topics (0%)","Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze.","Enable compression on topics to reduce storage usage and improve network performance" +"Infinite Retention Policy","pass","No topics have infinite retention policy enabled","Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy.","" \ No newline at end of file diff --git a/kafka-analysis/kafka-health-checks-1752105730445.csv b/kafka-analysis/kafka-health-checks-1752105730445.csv new file mode 100644 index 0000000..fac3ac0 --- /dev/null +++ b/kafka-analysis/kafka-health-checks-1752105730445.csv @@ -0,0 +1,16 @@ +"Health Check Results" +"Check Name","Status","Message","Description","Recommendation" +"Replication Factor vs Broker Count","pass","All topics have appropriate replication factor (≤ 1 brokers)","Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count.","" +"Topic Partition Distribution","pass","Good partition distribution: avg=1.0, min=1, max=1","Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions.","" +"Consumer Group Health","warning","1 consumer group(s) have no active members: perf-test-group","Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members.","Consider cleaning up unused consumer groups" +"Internal Topics Health","pass","All 1 internal topics are healthy","Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions.","" +"Under-Replicated Partitions","pass","All topics have the expected number of in-sync replicas","","" +"Min In-Sync Replicas Configuration","pass","All topics have appropriate min.insync.replicas configuration","","" +"Rack Awareness","warning","Rack awareness is not configured - no brokers have rack information","Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured.","Consider enabling rack awareness for better availability and fault tolerance" +"Replica Distribution","pass","Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51)","Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues.","" +"Metrics Configuration","warning","No JMX metrics configuration detected on any brokers","Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured.","Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis" +"Generic Kafka Logging Configuration","info","Generic Kafka logging configuration check","Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured.","Verify log4j configuration and log directory permissions in server.properties" +"Generic Kafka Authentication Configuration","fail","Unauthenticated access is enabled - this is a security risk","Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk).","Enable SASL or SSL authentication in server.properties for better security" +"Generic Kafka Quotas Configuration","warning","No quota configuration detected in Kafka cluster","Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available.","Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management" +"Payload Compression","warning","No compression detected on any of the 1 user topics (0%)","Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze.","Enable compression on topics to reduce storage usage and improve network performance" +"Infinite Retention Policy","pass","No topics have infinite retention policy enabled","Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy.","" \ No newline at end of file diff --git a/kafka-analysis/kafka-report-1752105684783.html b/kafka-analysis/kafka-report-1752105684783.html new file mode 100644 index 0000000..f94a5af --- /dev/null +++ b/kafka-analysis/kafka-report-1752105684783.html @@ -0,0 +1,850 @@ + + + + Modern Kafka Health Report + + + +

+
+ +
+ + +
+

Cluster Overview

+ +
+
+
+ Database + Vendor +
+
apache-kafka
+
+ +
+
+ Activity + Brokers +
+
1
+
+ +
+
+ Users + ZooKeepers +
+
1
+
+ +
+
+ Calendar + Generated +
+
7/10/2025, 3:01:24 AM
+
+
+ +
+

Topic Statistics

+
+ Total Topics + 2 +
+
+ User Topics + 1 +
+
+ Internal Topics + 1 +
+
+ Total Partitions + 51 +
+
+ Topics with Issues + 0 +
+
+
+
+ + +
+
+

Kafka Health Report

+

Comprehensive analysis of your Kafka cluster health and performance

+
+ + + +
+

Health Check Summary

+
+
+
14
+
Total Checks
+
+
+
7
+
Passed
+
+
+
5
+
Warnings
+
+
+
1
+
Failed
+
+
+
+ + +
+

Health Check Results

+ +
+ +
+
+
+ Failed +
+

Generic Kafka Authentication Configuration

+ Failed +
+ +
+ Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). +
+
+ Unauthenticated access is enabled - this is a security risk +
+ +
+

💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security

+
+
+ +
+
+
+ Warning +
+

Consumer Group Health

+ Warning +
+ +
+ Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. +
+
+ 1 consumer group(s) have no active members: perf-test-group +
+ +
+

💡 Recommendation: Consider cleaning up unused consumer groups

+
+
+ +
+
+
+ Warning +
+

Rack Awareness

+ Warning +
+ +
+ Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. +
+
+ Rack awareness is not configured - no brokers have rack information +
+ +
+

💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance

+
+
+ +
+
+
+ Warning +
+

Metrics Configuration

+ Warning +
+ +
+ Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. +
+
+ No JMX metrics configuration detected on any brokers +
+ +
+

💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis

+
+
+ +
+
+
+ Warning +
+

Generic Kafka Quotas Configuration

+ Warning +
+ +
+ Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. +
+
+ No quota configuration detected in Kafka cluster +
+ +
+

💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management

+
+
+ +
+
+
+ Warning +
+

Payload Compression

+ Warning +
+ +
+ Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. +
+
+ No compression detected on any of the 1 user topics (0%) +
+ +
+

💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance

+
+
+ +
+
+
+ Info +
+

Generic Kafka Logging Configuration

+ Info +
+ +
+ Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. +
+
+ Generic Kafka logging configuration check +
+ +
+

💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties

+
+
+ +
+
+
+ Passed +
+

Replication Factor vs Broker Count

+ Passed +
+ +
+ Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. +
+
+ All topics have appropriate replication factor (≤ 1 brokers) +
+ +
+ +
+
+
+ Passed +
+

Topic Partition Distribution

+ Passed +
+ +
+ Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. +
+
+ Good partition distribution: avg=1.0, min=1, max=1 +
+ +
+ +
+
+
+ Passed +
+

Internal Topics Health

+ Passed +
+ +
+ Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. +
+
+ All 1 internal topics are healthy +
+ +
+ +
+
+
+ Passed +
+

Under-Replicated Partitions

+ Passed +
+ +
+ All topics have the expected number of in-sync replicas +
+ +
+ +
+
+
+ Passed +
+

Min In-Sync Replicas Configuration

+ Passed +
+ +
+ All topics have appropriate min.insync.replicas configuration +
+ +
+ +
+
+
+ Passed +
+

Replica Distribution

+ Passed +
+ +
+ Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. +
+
+ Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) +
+ +
+ +
+
+
+ Passed +
+

Infinite Retention Policy

+ Passed +
+ +
+ Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. +
+
+ No topics have infinite retention policy enabled +
+ +
+ +
+
+ + + +
+

+ Generated by Superstream Kafka Analyzer +

+

+ Want to fix your Kafka issues automatically and continuously? Learn more +

+ + +
+ + Superstream - Fix your Kafka issues automatically + +
+ +
+ All rights reserved to Superstream Labs Inc 2025. You own your data and it is never shared with anyone, including us. +
+
+
+
+
+ + \ No newline at end of file diff --git a/kafka-analysis/kafka-report-1752105715380.html b/kafka-analysis/kafka-report-1752105715380.html new file mode 100644 index 0000000..1fc2ec5 --- /dev/null +++ b/kafka-analysis/kafka-report-1752105715380.html @@ -0,0 +1,850 @@ + + + + Modern Kafka Health Report + + + +
+
+ +
+ + +
+

Cluster Overview

+ +
+
+
+ Database + Vendor +
+
apache-kafka
+
+ +
+
+ Activity + Brokers +
+
1
+
+ +
+
+ Users + ZooKeepers +
+
1
+
+ +
+
+ Calendar + Generated +
+
7/10/2025, 3:01:55 AM
+
+
+ +
+

Topic Statistics

+
+ Total Topics + 2 +
+
+ User Topics + 1 +
+
+ Internal Topics + 1 +
+
+ Total Partitions + 51 +
+
+ Topics with Issues + 0 +
+
+
+
+ + +
+
+

Kafka Health Report

+

Comprehensive analysis of your Kafka cluster health and performance

+
+ + + +
+

Health Check Summary

+
+
+
14
+
Total Checks
+
+
+
7
+
Passed
+
+
+
5
+
Warnings
+
+
+
1
+
Failed
+
+
+
+ + +
+

Health Check Results

+ +
+ +
+
+
+ Failed +
+

Generic Kafka Authentication Configuration

+ Failed +
+ +
+ Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). +
+
+ Unauthenticated access is enabled - this is a security risk +
+ +
+

💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security

+
+
+ +
+
+
+ Warning +
+

Consumer Group Health

+ Warning +
+ +
+ Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. +
+
+ 1 consumer group(s) have no active members: perf-test-group +
+ +
+

💡 Recommendation: Consider cleaning up unused consumer groups

+
+
+ +
+
+
+ Warning +
+

Rack Awareness

+ Warning +
+ +
+ Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. +
+
+ Rack awareness is not configured - no brokers have rack information +
+ +
+

💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance

+
+
+ +
+
+
+ Warning +
+

Metrics Configuration

+ Warning +
+ +
+ Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. +
+
+ No JMX metrics configuration detected on any brokers +
+ +
+

💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis

+
+
+ +
+
+
+ Warning +
+

Generic Kafka Quotas Configuration

+ Warning +
+ +
+ Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. +
+
+ No quota configuration detected in Kafka cluster +
+ +
+

💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management

+
+
+ +
+
+
+ Warning +
+

Payload Compression

+ Warning +
+ +
+ Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. +
+
+ No compression detected on any of the 1 user topics (0%) +
+ +
+

💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance

+
+
+ +
+
+
+ Info +
+

Generic Kafka Logging Configuration

+ Info +
+ +
+ Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. +
+
+ Generic Kafka logging configuration check +
+ +
+

💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties

+
+
+ +
+
+
+ Passed +
+

Replication Factor vs Broker Count

+ Passed +
+ +
+ Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. +
+
+ All topics have appropriate replication factor (≤ 1 brokers) +
+ +
+ +
+
+
+ Passed +
+

Topic Partition Distribution

+ Passed +
+ +
+ Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. +
+
+ Good partition distribution: avg=1.0, min=1, max=1 +
+ +
+ +
+
+
+ Passed +
+

Internal Topics Health

+ Passed +
+ +
+ Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. +
+
+ All 1 internal topics are healthy +
+ +
+ +
+
+
+ Passed +
+

Under-Replicated Partitions

+ Passed +
+ +
+ All topics have the expected number of in-sync replicas +
+ +
+ +
+
+
+ Passed +
+

Min In-Sync Replicas Configuration

+ Passed +
+ +
+ All topics have appropriate min.insync.replicas configuration +
+ +
+ +
+
+
+ Passed +
+

Replica Distribution

+ Passed +
+ +
+ Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. +
+
+ Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) +
+ +
+ +
+
+
+ Passed +
+

Infinite Retention Policy

+ Passed +
+ +
+ Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. +
+
+ No topics have infinite retention policy enabled +
+ +
+ +
+
+ + + +
+

+ Generated by Superstream Kafka Analyzer +

+

+ Want to fix your Kafka issues automatically and continuously? Learn more +

+ + +
+ + Superstream - Fix your Kafka issues automatically + +
+ +
+ All rights reserved to Superstream Labs Inc 2025. You own your data and it is never shared with anyone, including us. +
+
+
+
+
+ + \ No newline at end of file diff --git a/kafka-analysis/kafka-report-1752105730445.html b/kafka-analysis/kafka-report-1752105730445.html new file mode 100644 index 0000000..5fd13c7 --- /dev/null +++ b/kafka-analysis/kafka-report-1752105730445.html @@ -0,0 +1,850 @@ + + + + Modern Kafka Health Report + + + +
+
+ +
+ + +
+

Cluster Overview

+ +
+
+
+ Database + Vendor +
+
apache-kafka
+
+ +
+
+ Activity + Brokers +
+
1
+
+ +
+
+ Users + ZooKeepers +
+
1
+
+ +
+
+ Calendar + Generated +
+
7/10/2025, 3:02:10 AM
+
+
+ +
+

Topic Statistics

+
+ Total Topics + 2 +
+
+ User Topics + 1 +
+
+ Internal Topics + 1 +
+
+ Total Partitions + 51 +
+
+ Topics with Issues + 0 +
+
+
+
+ + +
+
+

Kafka Health Report

+

Comprehensive analysis of your Kafka cluster health and performance

+
+ + + +
+

Health Check Summary

+
+
+
14
+
Total Checks
+
+
+
7
+
Passed
+
+
+
5
+
Warnings
+
+
+
1
+
Failed
+
+
+
+ + +
+

Health Check Results

+ +
+ +
+
+
+ Failed +
+

Generic Kafka Authentication Configuration

+ Failed +
+ +
+ Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). +
+
+ Unauthenticated access is enabled - this is a security risk +
+ +
+

💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security

+
+
+ +
+
+
+ Warning +
+

Consumer Group Health

+ Warning +
+ +
+ Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. +
+
+ 1 consumer group(s) have no active members: perf-test-group +
+ +
+

💡 Recommendation: Consider cleaning up unused consumer groups

+
+
+ +
+
+
+ Warning +
+

Rack Awareness

+ Warning +
+ +
+ Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. +
+
+ Rack awareness is not configured - no brokers have rack information +
+ +
+

💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance

+
+
+ +
+
+
+ Warning +
+

Metrics Configuration

+ Warning +
+ +
+ Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. +
+
+ No JMX metrics configuration detected on any brokers +
+ +
+

💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis

+
+
+ +
+
+
+ Warning +
+

Generic Kafka Quotas Configuration

+ Warning +
+ +
+ Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. +
+
+ No quota configuration detected in Kafka cluster +
+ +
+

💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management

+
+
+ +
+
+
+ Warning +
+

Payload Compression

+ Warning +
+ +
+ Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. +
+
+ No compression detected on any of the 1 user topics (0%) +
+ +
+

💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance

+
+
+ +
+
+
+ Info +
+

Generic Kafka Logging Configuration

+ Info +
+ +
+ Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. +
+
+ Generic Kafka logging configuration check +
+ +
+

💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties

+
+
+ +
+
+
+ Passed +
+

Replication Factor vs Broker Count

+ Passed +
+ +
+ Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. +
+
+ All topics have appropriate replication factor (≤ 1 brokers) +
+ +
+ +
+
+
+ Passed +
+

Topic Partition Distribution

+ Passed +
+ +
+ Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. +
+
+ Good partition distribution: avg=1.0, min=1, max=1 +
+ +
+ +
+
+
+ Passed +
+

Internal Topics Health

+ Passed +
+ +
+ Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. +
+
+ All 1 internal topics are healthy +
+ +
+ +
+
+
+ Passed +
+

Under-Replicated Partitions

+ Passed +
+ +
+ All topics have the expected number of in-sync replicas +
+ +
+ +
+
+
+ Passed +
+

Min In-Sync Replicas Configuration

+ Passed +
+ +
+ All topics have appropriate min.insync.replicas configuration +
+ +
+ +
+
+
+ Passed +
+

Replica Distribution

+ Passed +
+ +
+ Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. +
+
+ Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) +
+ +
+ +
+
+
+ Passed +
+

Infinite Retention Policy

+ Passed +
+ +
+ Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. +
+
+ No topics have infinite retention policy enabled +
+ +
+ +
+
+ + + +
+

+ Generated by Superstream Kafka Analyzer +

+

+ Want to fix your Kafka issues automatically and continuously? Learn more +

+ + +
+ + Superstream - Fix your Kafka issues automatically + +
+ +
+ All rights reserved to Superstream Labs Inc 2025. You own your data and it is never shared with anyone, including us. +
+
+
+
+
+ + \ No newline at end of file diff --git a/kafka-analysis/kafka-summary-1752105684783.txt b/kafka-analysis/kafka-summary-1752105684783.txt new file mode 100644 index 0000000..d7eb92b --- /dev/null +++ b/kafka-analysis/kafka-summary-1752105684783.txt @@ -0,0 +1,77 @@ +Kafka Analysis Summary +---------------------- +ZooKeepers: 1 +Brokers: 1 +Total Topics: 2 +User Topics: 1 +Internal Topics: 1 +Total Partitions: 51 +Topics with Issues: 0 + +Health Check Results +------------------- +Total Checks: 14 +✅ Passed: 7 +❌ Failed: 1 +⚠ī¸ Warnings: 5 + +✅ Replication Factor vs Broker Count + 📋 Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. + All topics have appropriate replication factor (≤ 1 brokers) + +✅ Topic Partition Distribution + 📋 Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. + Good partition distribution: avg=1.0, min=1, max=1 + +⚠ī¸ Consumer Group Health + 📋 Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. + 1 consumer group(s) have no active members: perf-test-group + 💡 Recommendation: Consider cleaning up unused consumer groups + +✅ Internal Topics Health + 📋 Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. + All 1 internal topics are healthy + +✅ Under-Replicated Partitions + All topics have the expected number of in-sync replicas + +✅ Min In-Sync Replicas Configuration + All topics have appropriate min.insync.replicas configuration + +⚠ī¸ Rack Awareness + 📋 Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. + Rack awareness is not configured - no brokers have rack information + 💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance + +✅ Replica Distribution + 📋 Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. + Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) + +⚠ī¸ Metrics Configuration + 📋 Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. + No JMX metrics configuration detected on any brokers + 💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis + +ℹī¸ Generic Kafka Logging Configuration + 📋 Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. + Generic Kafka logging configuration check + 💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties + +❌ Generic Kafka Authentication Configuration + 📋 Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). + Unauthenticated access is enabled - this is a security risk + 💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security + +⚠ī¸ Generic Kafka Quotas Configuration + 📋 Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. + No quota configuration detected in Kafka cluster + 💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management + +⚠ī¸ Payload Compression + 📋 Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. + No compression detected on any of the 1 user topics (0%) + 💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance + +✅ Infinite Retention Policy + 📋 Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. + No topics have infinite retention policy enabled diff --git a/kafka-analysis/kafka-summary-1752105715380.txt b/kafka-analysis/kafka-summary-1752105715380.txt new file mode 100644 index 0000000..d7eb92b --- /dev/null +++ b/kafka-analysis/kafka-summary-1752105715380.txt @@ -0,0 +1,77 @@ +Kafka Analysis Summary +---------------------- +ZooKeepers: 1 +Brokers: 1 +Total Topics: 2 +User Topics: 1 +Internal Topics: 1 +Total Partitions: 51 +Topics with Issues: 0 + +Health Check Results +------------------- +Total Checks: 14 +✅ Passed: 7 +❌ Failed: 1 +⚠ī¸ Warnings: 5 + +✅ Replication Factor vs Broker Count + 📋 Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. + All topics have appropriate replication factor (≤ 1 brokers) + +✅ Topic Partition Distribution + 📋 Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. + Good partition distribution: avg=1.0, min=1, max=1 + +⚠ī¸ Consumer Group Health + 📋 Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. + 1 consumer group(s) have no active members: perf-test-group + 💡 Recommendation: Consider cleaning up unused consumer groups + +✅ Internal Topics Health + 📋 Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. + All 1 internal topics are healthy + +✅ Under-Replicated Partitions + All topics have the expected number of in-sync replicas + +✅ Min In-Sync Replicas Configuration + All topics have appropriate min.insync.replicas configuration + +⚠ī¸ Rack Awareness + 📋 Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. + Rack awareness is not configured - no brokers have rack information + 💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance + +✅ Replica Distribution + 📋 Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. + Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) + +⚠ī¸ Metrics Configuration + 📋 Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. + No JMX metrics configuration detected on any brokers + 💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis + +ℹī¸ Generic Kafka Logging Configuration + 📋 Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. + Generic Kafka logging configuration check + 💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties + +❌ Generic Kafka Authentication Configuration + 📋 Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). + Unauthenticated access is enabled - this is a security risk + 💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security + +⚠ī¸ Generic Kafka Quotas Configuration + 📋 Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. + No quota configuration detected in Kafka cluster + 💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management + +⚠ī¸ Payload Compression + 📋 Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. + No compression detected on any of the 1 user topics (0%) + 💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance + +✅ Infinite Retention Policy + 📋 Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. + No topics have infinite retention policy enabled diff --git a/kafka-analysis/kafka-summary-1752105730445.txt b/kafka-analysis/kafka-summary-1752105730445.txt new file mode 100644 index 0000000..d7eb92b --- /dev/null +++ b/kafka-analysis/kafka-summary-1752105730445.txt @@ -0,0 +1,77 @@ +Kafka Analysis Summary +---------------------- +ZooKeepers: 1 +Brokers: 1 +Total Topics: 2 +User Topics: 1 +Internal Topics: 1 +Total Partitions: 51 +Topics with Issues: 0 + +Health Check Results +------------------- +Total Checks: 14 +✅ Passed: 7 +❌ Failed: 1 +⚠ī¸ Warnings: 5 + +✅ Replication Factor vs Broker Count + 📋 Checks if any topic has a replication factor greater than the number of brokers. Healthy: All topics have RF ≤ broker count. Failed: Any topic has RF > broker count. + All topics have appropriate replication factor (≤ 1 brokers) + +✅ Topic Partition Distribution + 📋 Checks if user topics have a balanced number of partitions. Healthy: Partition counts are similar. Warning: Large difference between min and max partitions. + Good partition distribution: avg=1.0, min=1, max=1 + +⚠ī¸ Consumer Group Health + 📋 Checks if all consumer groups have active members. Healthy: All groups have members. Warning: Some groups have no active members. + 1 consumer group(s) have no active members: perf-test-group + 💡 Recommendation: Consider cleaning up unused consumer groups + +✅ Internal Topics Health + 📋 Checks if all internal topics (names starting with __) have partitions > 0. Healthy: All internal topics have partitions. Failed: Any internal topic has 0 or missing partitions. + All 1 internal topics are healthy + +✅ Under-Replicated Partitions + All topics have the expected number of in-sync replicas + +✅ Min In-Sync Replicas Configuration + All topics have appropriate min.insync.replicas configuration + +⚠ī¸ Rack Awareness + 📋 Checks if rack awareness is configured in the cluster. Healthy: Rack awareness is configured. Warning: Rack awareness is not configured. + Rack awareness is not configured - no brokers have rack information + 💡 Recommendation: Consider enabling rack awareness for better availability and fault tolerance + +✅ Replica Distribution + 📋 Checks if data replicas are evenly distributed across all brokers. Healthy: Each broker carries a similar number of replicas. Warning/Failed: Some brokers carry significantly more replicas than others, which can cause performance issues. + Perfect replica balance: Each broker carries 51.0 replicas on average (range: 51-51) + +⚠ī¸ Metrics Configuration + 📋 Checks if monitoring metrics are properly configured. For AWS MSK: Checks Open Monitoring with Prometheus JMX exporter. For others: Checks JMX metrics configuration. Healthy: Metrics are enabled and accessible. Warning: Metrics are not configured or partially configured. + No JMX metrics configuration detected on any brokers + 💡 Recommendation: Enable JMX metrics on brokers for better monitoring, alerting, and performance analysis + +ℹī¸ Generic Kafka Logging Configuration + 📋 Checks if logging configuration is properly configured. For AWS MSK: Checks LoggingInfo configuration and CloudTrail. For Confluent Cloud/Aiven: Built-in logging is available. For others: Checks log4j configuration. Healthy: Logging is enabled and configured. Warning: Logging is not configured or partially configured. + Generic Kafka logging configuration check + 💡 Recommendation: Verify log4j configuration and log directory permissions in server.properties + +❌ Generic Kafka Authentication Configuration + 📋 Checks if unauthenticated access is enabled. For AWS MSK: Checks if SASL or SSL is configured. For Confluent Cloud/Aiven: Built-in authentication prevents unauthenticated access. For others: Checks if SASL or SSL is configured. Healthy: Authentication is enabled (no unauthenticated access). Failed: Unauthenticated access is enabled (security risk). + Unauthenticated access is enabled - this is a security risk + 💡 Recommendation: Enable SASL or SSL authentication in server.properties for better security + +⚠ī¸ Generic Kafka Quotas Configuration + 📋 Checks if Kafka quotas are configured and being used. For AWS MSK: Checks quota configuration via AWS console/CLI. For Confluent Cloud/Aiven: Built-in quota management is available. For others: Checks server.properties and kafka-configs.sh for quota settings. Healthy: Quotas are configured and managed. Info: Quotas configuration check available. + No quota configuration detected in Kafka cluster + 💡 Recommendation: Configure quotas in server.properties or use kafka-configs.sh to set client quotas for better resource management + +⚠ī¸ Payload Compression + 📋 Checks if payload compression is enabled on user topics. Analyzes compression.type, compression, and producer.compression.type configurations. Healthy: All user topics have compression enabled (100%). Warning: Some or no topics have compression enabled (<100%). Info: No user topics to analyze. + No compression detected on any of the 1 user topics (0%) + 💡 Recommendation: Enable compression on topics to reduce storage usage and improve network performance + +✅ Infinite Retention Policy + 📋 Checks if any topics have infinite retention policy enabled (retention.ms = infinite). Healthy: No topics have infinite retention. Warning: Some topics have infinite retention policy (bad practice). Info: Unable to verify retention policy. + No topics have infinite retention policy enabled diff --git a/kraft-config.json b/kraft-config.json new file mode 100644 index 0000000..e365679 --- /dev/null +++ b/kraft-config.json @@ -0,0 +1,15 @@ +{ + "kafka": { + "bootstrap_servers": ["localhost:29092"], + "clientId": "superstream-analyzer", + "vendor": "apache-kafka", + "useSasl": false + }, + "file": { + "outputDir": "./kafka-analysis", + "formats": ["json", "html", "csv", "txt"], + "includeMetadata": true, + "includeTimestamp": true + }, + "email": "test@example.com" +} diff --git a/src/cli.js b/src/cli.js index 4780b15..067bfc6 100644 --- a/src/cli.js +++ b/src/cli.js @@ -46,6 +46,23 @@ class CLI { this.config = config; + // Normalize brokers/bootstrap_servers to array format + if (this.config.kafka) { + // Handle both 'bootstrap_servers' and 'brokers' fields + const brokers = this.config.kafka.bootstrap_servers || this.config.kafka.brokers; + + if (brokers) { + if (Array.isArray(brokers)) { + this.config.kafka.brokers = brokers; + } else if (typeof brokers === 'string') { + this.config.kafka.brokers = brokers.split(',').map(broker => broker.trim()); + } + + // Remove bootstrap_servers field if it exists, standardize on 'brokers' + delete this.config.kafka.bootstrap_servers; + } + } + // Add email if not present in config file if (!this.config.email) { this.config.email = ''; @@ -261,6 +278,10 @@ class CLI { // Build kafka config this.config.kafka = { ...kafkaAnswers, + // Convert brokers string to array + brokers: Array.isArray(kafkaAnswers.brokers) + ? kafkaAnswers.brokers + : kafkaAnswers.brokers.split(',').map(broker => broker.trim()), vendor: vendorAnswer.vendor, useSasl: !!saslConfig, sasl: saslConfig @@ -345,6 +366,12 @@ class CLI { } else { console.log(chalk.gray('Debug: No config file specified, using interactive mode')); await this.promptForConfig(); + + // Override with command line options if provided + if (this.options.bootstrapServers) { + console.log(chalk.gray(`Debug: Overriding brokers with command line option: ${this.options.bootstrapServers}`)); + this.config.kafka.brokers = this.options.bootstrapServers.split(',').map(broker => broker.trim()); + } } // Initialize services without validation