""" Phase 3A: Incident Telemetry & Alert Fidelity Optimization Subtopics: 1. Control-M job failure pattern clustering using KMeans 2. Alert fidelity scoring via close_notes semantic parsing 3. Incident volume forecasting with seasonal decomposition 4. Priority distribution calibration analysis """ import pandas as pd import numpy as np import plotly.express as px import plotly.graph_objects as go from plotly.subplots import make_subplots from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler, LabelEncoder from statsmodels.tsa.seasonal import STL import warnings warnings.filterwarnings('ignore') # Load cleaned data inc = pd.read_parquet("temp_files/incidents_cleaned.parquet") cr = pd.read_parquet("temp_files/cr_cleaned.parquet") sr = pd.read_parquet("temp_files/sr_cleaned.parquet") print("=" * 80) print("PHASE 3A: INCIDENT TELEMETRY & ALERT FIDELITY OPTIMIZATION") print("=" * 80) # ============================================================ # 1. CONTROL-M JOB FAILURE PATTERN CLUSTERING (KMeans) # ============================================================ print("\n--- 1. Control-M Job Failure Pattern Clustering ---") # Filter Control-M related incidents control_m_mask = inc['short_description'].str.contains('Control-M', case=False, na=False) cm_inc = inc[control_m_mask].copy() print(f" Control-M incidents: {len(cm_inc)}") if len(cm_inc) > 0: # Feature engineering for clustering cm_inc['hour'] = cm_inc['opened_at'].dt.hour cm_inc['day_of_week_num'] = cm_inc['opened_at'].dt.dayofweek cm_inc['is_weekend'] = (cm_inc['opened_at'].dt.dayofweek >= 5).astype(int) # Encode categorical features le_cat = LabelEncoder() cm_inc['category_encoded'] = le_cat.fit_transform(cm_inc['Category'].astype(str)) le_ci = LabelEncoder() cm_inc['ci_encoded'] = le_ci.fit_transform(cm_inc['cmdb_ci'].astype(str)) # Features for clustering features = ['hour', 'day_of_week_num', 'is_weekend', 'category_encoded', 'ci_encoded'] X = cm_inc[features].fillna(0) # Standardize scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # KMeans clustering (optimal k via elbow method - using k=4 based on domain knowledge) kmeans = KMeans(n_clusters=4, random_state=42, n_init=10) cm_inc['cluster'] = kmeans.fit_predict(X_scaled) print(f" Cluster distribution: {cm_inc['cluster'].value_counts().to_dict()}") # Analyze clusters cluster_summary = cm_inc.groupby('cluster').agg({ 'hour': 'mean', 'day_of_week_num': 'mean', 'Category': lambda x: x.mode()[0] if not x.mode().empty else 'Unknown', 'cmdb_ci': 'nunique', 'number': 'count' }).rename(columns={'number': 'count', 'cmdb_ci': 'unique_cis'}) print(cluster_summary) # Plot: Cluster scatter (hour vs day_of_week) fig_cm = px.scatter( cm_inc, x='hour', y='day_of_week_num', color='cluster', hover_data=['Category', 'cmdb_ci'], title='Control-M Job Failure Clusters (Hour vs Day of Week)', labels={'hour': 'Hour of Day', 'day_of_week_num': 'Day of Week (0=Mon)'}, color_continuous_scale='Viridis' ) fig_cm.update_layout(template='plotly_white') fig_cm.write_html("assets/images/html/incident_controlm_clusters.html") fig_cm.write_image("assets/images/png/incident_controlm_clusters.png", width=1000, height=600, scale=2) # Plot: Cluster characteristics bar chart cluster_chars = cm_inc.groupby(['cluster', 'Category']).size().unstack(fill_value=0) fig_cm2 = go.Figure() for cat in cluster_chars.columns[:5]: # Top 5 categories fig_cm2.add_trace(go.Bar( name=cat, x=cluster_chars.index, y=cluster_chars[cat] )) fig_cm2.update_layout( barmode='group', title='Control-M Cluster: Category Composition', xaxis_title='Cluster', yaxis_title='Count', template='plotly_white' ) fig_cm2.write_html("assets/images/html/incident_controlm_cluster_categories.html") fig_cm2.write_image("assets/images/png/incident_controlm_cluster_categories.png", width=1000, height=600, scale=2) # ============================================================ # 2. ALERT FIDELITY SCORING VIA CLOSE_NOTES SEMANTIC PARSING # ============================================================ print("\n--- 2. Alert Fidelity Scoring ---") # Define sentinel phrases for non-actionable alerts sentinel_phrases = { 'dummy_warning': ['dummy', 'test alert', 'test incident', 'false alert'], 'duplicate': ['duplicate', 'duplicate incident', 'dup', 'same as'], 'refresh_in_progress': ['refresh in progress', 'refresh ongoing', 'system refresh'], 'maintenance': ['maintenance', 'scheduled maintenance', 'planned maintenance'], 'self_resolved': ['self resolved', 'auto resolved', 'resolved automatically', 'cleared itself'], 'no_issue': ['no issue found', 'false positive', 'not an issue', 'working as expected'], 'reboot_required': ['reboot', 'restart required', 'bounced'], } def classify_fidelity(row): notes = str(row['close_notes']).lower() desc = str(row['short_description']).lower() combined = notes + ' ' + desc for category, phrases in sentinel_phrases.items(): if any(phrase in combined for phrase in phrases): return category return 'actionable' inc['fidelity_class'] = inc.apply(classify_fidelity, axis=1) fidelity_counts = inc['fidelity_class'].value_counts() print(f" Fidelity distribution:\n{fidelity_counts}") # Calculate fidelity score (actionable / total) actionable_pct = fidelity_counts.get('actionable', 0) / len(inc) * 100 print(f" Actionable alert rate: {actionable_pct:.1f}%") print(f" Non-actionable alert rate: {100 - actionable_pct:.1f}%") # Plot: Fidelity class distribution fig_fid = px.pie( fidelity_counts.reset_index(), names='fidelity_class', values='count', title='Alert Fidelity Classification Distribution', color_discrete_sequence=px.colors.qualitative.Set2 ) fig_fid.update_traces(textinfo='percent+label') fig_fid.update_layout(template='plotly_white') fig_fid.write_html("assets/images/html/incident_fidelity_distribution.html") fig_fid.write_image("assets/images/png/incident_fidelity_distribution.png", width=900, height=600, scale=2) # Plot: Fidelity by category (top categories) fidelity_by_cat = inc.groupby(['Category', 'fidelity_class']).size().unstack(fill_value=0) fidelity_by_cat['total'] = fidelity_by_cat.sum(axis=1) fidelity_by_cat = fidelity_by_cat.sort_values('total', ascending=False).head(15) fidelity_by_cat = fidelity_by_cat.drop(columns=['total']) fig_fid2 = go.Figure() for cls in fidelity_by_cat.columns: fig_fid2.add_trace(go.Bar(name=cls, x=fidelity_by_cat.index, y=fidelity_by_cat[cls])) fig_fid2.update_layout( barmode='stack', title='Alert Fidelity by Incident Category (Top 15)', xaxis_title='Category', yaxis_title='Count', template='plotly_white', xaxis_tickangle=45 ) fig_fid2.write_html("assets/images/html/incident_fidelity_by_category.html") fig_fid2.write_image("assets/images/png/incident_fidelity_by_category.png", width=1200, height=600, scale=2) # ============================================================ # 3. INCIDENT VOLUME FORECASTING WITH SEASONAL DECOMPOSITION # ============================================================ print("\n--- 3. Incident Volume Forecasting (STL Decomposition) ---") # Daily incident counts daily_counts = inc.groupby(inc['opened_at'].dt.date).size().reset_index(name='count') daily_counts['opened_at'] = pd.to_datetime(daily_counts['opened_at']) daily_counts = daily_counts.set_index('opened_at').asfreq('D').fillna(0) daily_counts = daily_counts['count'] # Ensure minimum length for STL if len(daily_counts) >= 14: # STL decomposition (weekly seasonality = 7 days) stl = STL(daily_counts, period=7, robust=True) result = stl.fit() # Plot decomposition fig_stl = make_subplots(rows=4, cols=1, shared_xaxes=True, subplot_titles=('Observed', 'Trend', 'Seasonal', 'Residual')) fig_stl.add_trace(go.Scatter(x=daily_counts.index, y=result.observed, mode='lines', name='Observed'), row=1, col=1) fig_stl.add_trace(go.Scatter(x=daily_counts.index, y=result.trend, mode='lines', name='Trend', line=dict(color='red')), row=2, col=1) fig_stl.add_trace(go.Scatter(x=daily_counts.index, y=result.seasonal, mode='lines', name='Seasonal', line=dict(color='green')), row=3, col=1) fig_stl.add_trace(go.Scatter(x=daily_counts.index, y=result.resid, mode='lines', name='Residual', line=dict(color='purple')), row=4, col=1) fig_stl.update_layout(height=900, title_text='Daily Incident Volume STL Decomposition', template='plotly_white', showlegend=False) fig_stl.write_html("assets/images/html/incident_stl_decomposition.html") fig_stl.write_image("assets/images/png/incident_stl_decomposition.png", width=1200, height=900, scale=2) # Forecast next 14 days using trend + seasonal trend_forecast = result.trend.dropna() seasonal_avg = result.seasonal.groupby(result.seasonal.index.dayofweek).mean() last_date = daily_counts.index[-1] forecast_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), periods=14, freq='D') forecast_values = [] for date in forecast_dates: dow = date.dayofweek # Use last known trend value + seasonal component trend_val = trend_forecast.iloc[-1] if len(trend_forecast) > 0 else daily_counts.mean() seasonal_val = seasonal_avg.get(dow, 0) forecast_values.append(max(0, trend_val + seasonal_val)) forecast_df = pd.DataFrame({'date': forecast_dates, 'forecast': forecast_values}) # Plot forecast fig_fc = go.Figure() fig_fc.add_trace(go.Scatter(x=daily_counts.index, y=daily_counts.values, mode='lines', name='Historical', line=dict(color='blue'))) fig_fc.add_trace(go.Scatter(x=forecast_df['date'], y=forecast_df['forecast'], mode='lines', name='Forecast', line=dict(color='red', dash='dash'))) fig_fc.update_layout( title='Incident Volume Forecast (14-Day)', xaxis_title='Date', yaxis_title='Daily Incident Count', template='plotly_white' ) fig_fc.write_html("assets/images/html/incident_volume_forecast.html") fig_fc.write_image("assets/images/png/incident_volume_forecast.png", width=1200, height=500, scale=2) print(f" Mean daily volume: {daily_counts.mean():.1f}") print(f" Trend direction: {'Increasing' if result.trend.dropna().iloc[-1] > result.trend.dropna().iloc[0] else 'Decreasing'}") print(f" Forecast next 14 days mean: {np.mean(forecast_values):.1f}") # Hourly pattern for follow-the-sun analysis hourly_counts = inc.groupby('hour').size().reset_index(name='count') fig_hourly = px.line(hourly_counts, x='hour', y='count', markers=True, title='Incident Arrival Pattern by Hour (UTC) - Follow-the-Sun Analysis', labels={'hour': 'Hour (UTC)', 'count': 'Incident Count'}) fig_hourly.add_vrect(x0=0, x1=8, fillcolor="blue", opacity=0.1, layer="below", line_width=0, annotation_text="EMEA Night", annotation_position="top left") fig_hourly.add_vrect(x0=8, x1=16, fillcolor="green", opacity=0.1, layer="below", line_width=0, annotation_text="EMEA Day / NA Morning", annotation_position="top left") fig_hourly.add_vrect(x0=16, x1=24, fillcolor="orange", opacity=0.1, layer="below", line_width=0, annotation_text="NA Day / EMEA Evening", annotation_position="top left") fig_hourly.update_layout(template='plotly_white') fig_hourly.write_html("assets/images/html/incident_hourly_follothesun.html") fig_hourly.write_image("assets/images/png/incident_hourly_follothesun.png", width=1100, height=500, scale=2) # ============================================================ # 4. PRIORITY DISTRIBUTION CALIBRATION ANALYSIS # ============================================================ print("\n--- 4. Priority Distribution Calibration ---") # Since we don't have closed_at, we'll use category severity as proxy # and analyze priority vs category patterns priority_cat = inc.groupby(['priority', 'Category']).size().unstack(fill_value=0) priority_cat_pct = priority_cat.div(priority_cat.sum(axis=1), axis=0) * 100 # Top categories by priority fig_prio = go.Figure() for prio in ['Priority 2', 'Priority 3', 'Priority 4']: if prio in priority_cat.index: top_cats = priority_cat.loc[prio].sort_values(ascending=False).head(10) fig_prio.add_trace(go.Bar(name=prio, x=top_cats.index, y=top_cats.values)) fig_prio.update_layout( barmode='group', title='Top Categories by Priority Level', xaxis_title='Category', yaxis_title='Count', template='plotly_white', xaxis_tickangle=45 ) fig_prio.write_html("assets/images/html/incident_priority_category.html") fig_prio.write_image("assets/images/png/incident_priority_category.png", width=1200, height=600, scale=2) # Priority by region prio_region = inc.groupby(['region', 'priority']).size().unstack(fill_value=0) prio_region_pct = prio_region.div(prio_region.sum(axis=1), axis=0) * 100 fig_prio2 = go.Figure() for prio in ['Priority 2', 'Priority 3', 'Priority 4']: if prio in prio_region_pct.columns: fig_prio2.add_trace(go.Bar(name=prio, x=prio_region_pct.index, y=prio_region_pct[prio])) fig_prio2.update_layout( barmode='group', title='Priority Distribution by Region (%)', xaxis_title='Region', yaxis_title='Percentage', template='plotly_white' ) fig_prio2.write_html("assets/images/html/incident_priority_region.html") fig_prio2.write_image("assets/images/png/incident_priority_region.png", width=900, height=500, scale=2) # Priority calibration assessment print(f" Priority 2 (High): {len(inc[inc['priority'] == 'Priority 2'])} ({len(inc[inc['priority'] == 'Priority 2'])/len(inc)*100:.1f}%)") print(f" Priority 3 (Medium): {len(inc[inc['priority'] == 'Priority 3'])} ({len(inc[inc['priority'] == 'Priority 3'])/len(inc)*100:.1f}%)") print(f" Priority 4 (Low): {len(inc[inc['priority'] == 'Priority 4'])} ({len(inc[inc['priority'] == 'Priority 4'])/len(inc)*100:.1f}%)") # Control-M vs non-Control-M priority cm_prio = inc[inc['short_description'].str.contains('Control-M', case=False, na=False)]['priority'].value_counts(normalize=True) * 100 non_cm_prio = inc[~inc['short_description'].str.contains('Control-M', case=False, na=False)]['priority'].value_counts(normalize=True) * 100 print(f" Control-M Priority 2 rate: {cm_prio.get('Priority 2', 0):.1f}%") print(f" Non-Control-M Priority 2 rate: {non_cm_prio.get('Priority 2', 0):.1f}%") # Save telemetry results telemetry_results = { 'control_m_incidents': len(cm_inc) if 'cm_inc' in locals() else 0, 'fidelity_actionable_pct': actionable_pct, 'fidelity_non_actionable_pct': 100 - actionable_pct, 'priority_p2_pct': len(inc[inc['priority'] == 'Priority 2'])/len(inc)*100, 'priority_p3_pct': len(inc[inc['priority'] == 'Priority 3'])/len(inc)*100, 'priority_p4_pct': len(inc[inc['priority'] == 'Priority 4'])/len(inc)*100, 'mean_daily_incidents': daily_counts.mean() if 'daily_counts' in locals() else 0, } pd.Series(telemetry_results).to_json("temp_files/telemetry_results.json") print("\nPhase 3A complete. All telemetry plots saved.")