Add IPADDRESSV4 export and update data assets

apps/sector_kml_generator.py

@@ -182,7 +182,7 @@ def _show_site_position_map(
     fig = go.Figure()
     mode = "markers+text" if show_labels else "markers"
     fig.add_trace(
-        go.Scattermapbox(
+        go.Scattermap(
             lat=map_df[lat_col],
             lon=map_df[lon_col],
             mode=mode,
@@ -197,12 +197,12 @@ def _show_site_position_map(
 
     fig.update_layout(
         title=title,
-        mapbox_style="open-street-map",
-        mapbox_center={
+        map_style="open-street-map",
+        map_center={
             "lat": float(map_df[lat_col].mean()),
             "lon": float(map_df[lon_col].mean()),
         },
-        mapbox_zoom=_estimate_zoom(map_df, lat_col, lon_col),
+        map_zoom=_estimate_zoom(map_df, lat_col, lon_col),
         height=620,
         margin={"r": 0, "t": 45, "l": 0, "b": 0},
     )
@@ -234,7 +234,7 @@ def _show_sector_map(df: pd.DataFrame, show_labels: bool = True) -> None:
         lats = [coord[1] for coord in coords]
         rgba = kml_color_to_rgba(row["color"])
         fig.add_trace(
-            go.Scattermapbox(
+            go.Scattermap(
                 lon=lons,
                 lat=lats,
                 mode="lines",
@@ -251,7 +251,7 @@ def _show_sector_map(df: pd.DataFrame, show_labels: bool = True) -> None:
     site_df = map_df.drop_duplicates(subset=["code"]).copy()
     mode = "markers+text" if show_labels else "markers"
     fig.add_trace(
-        go.Scattermapbox(
+        go.Scattermap(
             lat=site_df["Latitude"],
             lon=site_df["Longitude"],
             mode=mode,
@@ -266,12 +266,12 @@ def _show_sector_map(df: pd.DataFrame, show_labels: bool = True) -> None:
 
     fig.update_layout(
         title="Sector map preview",
-        mapbox_style="open-street-map",
-        mapbox_center={
+        map_style="open-street-map",
+        map_center={
             "lat": float(map_df["Latitude"].mean()),
             "lon": float(map_df["Longitude"].mean()),
         },
-        mapbox_zoom=_estimate_zoom(map_df, "Latitude", "Longitude"),
+        map_zoom=_estimate_zoom(map_df, "Latitude", "Longitude"),
         height=650,
         margin={"r": 0, "t": 45, "l": 0, "b": 0},
     )

data/kpi_health_check_profiles/Profil_1.json

@@ -1,7 +1,8 @@
 {
-  "name": "Profil_1",
-  "saved_at": "2025-12-13T13:16:45.937845Z",
+  "name": "profil_1",
+  "saved_at": "2026-03-30T10:24:07.600473Z",
   "config": {
+    "granularity": "Hourly",
     "analysis_range": [
       null,
       null
@@ -10,23 +11,31 @@
     "recent_days": 7,
     "rel_threshold_pct": 10.0,
     "min_consecutive_days": 3,
+    "only_complaint_sites": false,
     "min_criticality": 0,
     "min_anomaly_score": 0,
     "city_filter": "",
     "top_rat_filter": [
       "2G",
       "3G",
-      "LTE"
+      "LTE",
+      "TWAMP"
     ],
     "top_status_filter": [
       "DEGRADED",
       "PERSISTENT_DEGRADED"
     ],
-    "preset_selected": "presets_1.json",
+    "preset_selected": "presets_1",
     "drilldown": {
-      "site_code": 2130,
       "rat": "LTE",
-      "kpi": ""
+      "site_value": "",
+      "kpi_group": "All (selected KPIs)",
+      "kpi_group_mode": "",
+      "kpi": "% MIMO RI 2",
+      "compare_kpis": [],
+      "compare_norm": "None",
+      "show_sla": true,
+      "corr_window": "Recent"
     }
   }
 }

queries/process_all_db.py

@@ -1,6 +1,7 @@
 from queries.process_atoll_db import process_data_for_atoll
 from queries.process_gsm import combined_gsm_database, gsm_analaysis
 from queries.process_invunit import process_invunit_data, process_invunit_number_data
+from queries.process_ipaddressv4 import process_ipaddressv4_data
 from queries.process_lte import lte_fdd_analaysis, lte_tdd_analaysis, process_lte_data
 from queries.process_mrbts import process_mrbts_data
 from queries.process_nice_db import process_data_for_nice
@@ -33,21 +34,11 @@ def all_dbs(filepath: str):
 def process_all_tech_db(filepath: str):
     all_dbs(filepath)
     site_db()
+    process_ipaddressv4_data(filepath)
 
     UtilsVars.final_all_database = convert_database_dfs(
         UtilsVars.all_db_dfs,
-        [
-            "GSM",
-            "MAL",
-            "TRX",
-            "WCDMA",
-            "LTE_FDD",
-            "LTE_TDD",
-            "MRBTS",
-            "INVUNIT",
-            "INVUNIT_NUMBER",
-            "SITE",
-        ],
+        UtilsVars.all_db_dfs_names,
     )
 
 
@@ -64,20 +55,10 @@ def process_all_tech_db_with_stats(
     )
     lte_fdd_analaysis(filepath)
     lte_tdd_analaysis(filepath)
+    process_ipaddressv4_data(filepath)
     UtilsVars.final_all_database = convert_database_dfs(
         UtilsVars.all_db_dfs,
-        [
-            "GSM",
-            "MAL",
-            "TRX",
-            "WCDMA",
-            "LTE_FDD",
-            "LTE_TDD",
-            "MRBTS",
-            "INVUNIT",
-            "INVUNIT_NUMBER",
-            "SITE",
-        ],
+        UtilsVars.all_db_dfs_names,
     )
 
 

queries/process_ipaddressv4.py

@@ -0,0 +1,57 @@
+import pandas as pd
+
+from utils.dump_excel import read_dump_excel
+from utils.utils_vars import UtilsVars
+
+IPADDRESSV4_SHEET_NAME = "IPADDRESSV4"
+IPADDRESSV4_COLUMNS = ["MRBTS", "localIpAddr"]
+
+
+def _sheet_exists(file_path, sheet_name: str) -> bool:
+    if hasattr(file_path, "seek"):
+        file_path.seek(0)
+
+    with pd.ExcelFile(file_path, engine="calamine") as workbook:
+        exists = sheet_name in workbook.sheet_names
+
+    if hasattr(file_path, "seek"):
+        file_path.seek(0)
+
+    return exists
+
+
+def _normalize_mrbts(series: pd.Series) -> pd.Series:
+    normalized = series.astype("string").str.strip()
+    return normalized.str.replace(r"\.0+$", "", regex=True)
+
+
+def process_ipaddressv4_data(file_path: str) -> pd.DataFrame:
+    if not _sheet_exists(file_path, IPADDRESSV4_SHEET_NAME):
+        return pd.DataFrame(columns=IPADDRESSV4_COLUMNS)
+
+    dfs = read_dump_excel(
+        file_path,
+        sheet_name=[IPADDRESSV4_SHEET_NAME],
+        expected_columns=IPADDRESSV4_COLUMNS,
+    )
+    df_ipaddressv4 = dfs[IPADDRESSV4_SHEET_NAME].copy()
+    df_ipaddressv4.columns = df_ipaddressv4.columns.str.replace(r"[ ]", "", regex=True)
+
+    missing_columns = [
+        column for column in IPADDRESSV4_COLUMNS if column not in df_ipaddressv4.columns
+    ]
+    if missing_columns:
+        raise ValueError(
+            "IPADDRESSV4 sheet is missing required columns: "
+            + ", ".join(missing_columns)
+        )
+
+    df_ipaddressv4 = df_ipaddressv4[IPADDRESSV4_COLUMNS].copy()
+    df_ipaddressv4["MRBTS"] = _normalize_mrbts(df_ipaddressv4["MRBTS"])
+    df_ipaddressv4["localIpAddr"] = (
+        df_ipaddressv4["localIpAddr"].astype("string").str.strip()
+    )
+
+    UtilsVars.all_db_dfs.append(df_ipaddressv4)
+    UtilsVars.all_db_dfs_names.append(IPADDRESSV4_SHEET_NAME)
+    return df_ipaddressv4

tests/test_process_ipaddressv4.py

@@ -0,0 +1,94 @@
+import pandas as pd
+import pytest
+
+from queries.process_ipaddressv4 import process_ipaddressv4_data
+from utils.utils_vars import UtilsVars
+
+
+def setup_function():
+    UtilsVars.all_db_dfs = []
+    UtilsVars.all_db_dfs_names = []
+
+
+def test_process_ipaddressv4_data_keeps_only_mrbts_and_local_ip(monkeypatch):
+    dump_dfs = {
+        "IPADDRESSV4": pd.DataFrame(
+            {
+                " MRBTS ": [" 12345.0 ", "67890"],
+                "local Ip Addr": [" 10.1.1.1 ", "10.2.2.2"],
+                "extraColumn": ["ignored", "ignored"],
+            }
+        )
+    }
+
+    monkeypatch.setattr("queries.process_ipaddressv4._sheet_exists", lambda *args: True)
+    monkeypatch.setattr(
+        "queries.process_ipaddressv4.read_dump_excel", lambda *args, **kwargs: dump_dfs
+    )
+
+    df = process_ipaddressv4_data("dummy.xlsb")
+
+    assert list(df.columns) == ["MRBTS", "localIpAddr"]
+    assert list(df["MRBTS"]) == ["12345", "67890"]
+    assert list(df["localIpAddr"]) == ["10.1.1.1", "10.2.2.2"]
+    assert UtilsVars.all_db_dfs_names == ["IPADDRESSV4"]
+    assert UtilsVars.all_db_dfs[0].equals(df)
+
+
+def test_process_ipaddressv4_data_skips_when_sheet_is_absent(monkeypatch):
+    monkeypatch.setattr("queries.process_ipaddressv4._sheet_exists", lambda *args: False)
+
+    df = process_ipaddressv4_data("dummy.xlsb")
+
+    assert list(df.columns) == ["MRBTS", "localIpAddr"]
+    assert df.empty
+    assert UtilsVars.all_db_dfs == []
+    assert UtilsVars.all_db_dfs_names == []
+
+
+def test_process_ipaddressv4_data_requires_local_ip_column(monkeypatch):
+    dump_dfs = {"IPADDRESSV4": pd.DataFrame({"MRBTS": ["12345"]})}
+
+    monkeypatch.setattr("queries.process_ipaddressv4._sheet_exists", lambda *args: True)
+    monkeypatch.setattr(
+        "queries.process_ipaddressv4.read_dump_excel", lambda *args, **kwargs: dump_dfs
+    )
+
+    with pytest.raises(ValueError, match="localIpAddr"):
+        process_ipaddressv4_data("dummy.xlsb")
+
+
+def test_process_all_tech_db_exports_ipaddressv4_as_final_extra_sheet(monkeypatch):
+    from queries import process_all_db
+
+    captured = {}
+
+    def fake_all_dbs(filepath):
+        UtilsVars.all_db_dfs = [pd.DataFrame({"source": ["base"]})]
+        UtilsVars.all_db_dfs_names = ["GSM"]
+
+    def fake_site_db():
+        UtilsVars.all_db_dfs.append(pd.DataFrame({"source": ["site"]}))
+        UtilsVars.all_db_dfs_names.append("SITE")
+
+    def fake_ipaddressv4(filepath):
+        UtilsVars.all_db_dfs.append(pd.DataFrame({"MRBTS": ["12345"]}))
+        UtilsVars.all_db_dfs_names.append("IPADDRESSV4")
+
+    def fake_convert_database_dfs(dfs, sheet_names):
+        captured["dfs"] = dfs
+        captured["sheet_names"] = sheet_names
+        return b"excel"
+
+    monkeypatch.setattr(process_all_db, "all_dbs", fake_all_dbs)
+    monkeypatch.setattr(process_all_db, "site_db", fake_site_db)
+    monkeypatch.setattr(process_all_db, "process_ipaddressv4_data", fake_ipaddressv4)
+    monkeypatch.setattr(
+        process_all_db, "convert_database_dfs", fake_convert_database_dfs
+    )
+
+    process_all_db.process_all_tech_db("dummy.xlsb")
+
+    assert UtilsVars.final_all_database == b"excel"
+    assert captured["sheet_names"] == ["GSM", "SITE", "IPADDRESSV4"]
+    assert len(captured["dfs"]) == 3