Reuse parsed database results across generators

queries/process_all_db.py

@@ -8,12 +8,10 @@ from queries.process_nice_db import process_data_for_nice
 from queries.process_site_db import site_db
 from queries.process_wcdma import process_wcdma_data, wcdma_analaysis
 from utils.convert_to_excel import convert_database_dfs, convert_dfs
-from utils.dump_excel import clear_dump_excel_cache
 from utils.utils_vars import UtilsVars
 
 
 def clear_all_dbs():
-    clear_dump_excel_cache()
     UtilsVars.all_db_dfs.clear()
     UtilsVars.all_db_dfs_names.clear()
     UtilsVars.gsm_dfs.clear()

queries/process_gsm.py

@@ -7,6 +7,7 @@ from utils.config_band import bcf_band, config_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.dump_excel import read_dump_excel
 from utils.kml_creator import generate_kml_from_df
+from utils.processing_cache import get_or_build_processed, remember_processed
 from utils.utils_vars import (
     GsmAnalysisData,
     UtilsVars,
@@ -93,7 +94,7 @@ def compare_trx_tch_versus_mal(tch1, tch2):
     return set1 == set2
 
 
-def process_gsm_data(
+def _build_gsm_data(
     file_path: str,
     df_trx: pd.DataFrame | None = None,
     df_mal: pd.DataFrame | None = None,
@@ -201,13 +202,44 @@ def process_gsm_data(
     return df_2g
 
 
-def combined_gsm_database(file_path: str):
+def process_gsm_data(
+    file_path: str,
+    df_trx: pd.DataFrame | None = None,
+    df_mal: pd.DataFrame | None = None,
+) -> pd.DataFrame:
+    """
+    Process GSM data.
+
+    Provided TRX/MAL dataframes are used directly by the All DB path to avoid
+    recomputing intermediate sheets. Plain calls are cached per dump/config.
+    """
+    if df_trx is not None or df_mal is not None:
+        return _build_gsm_data(file_path, df_trx=df_trx, df_mal=df_mal)
+
+    return get_or_build_processed(
+        file_path,
+        "gsm_data",
+        lambda: _build_gsm_data(file_path),
+    )
+
+
+def _build_combined_gsm_database(file_path: str) -> list[pd.DataFrame]:
     df_bts = process_small_bts_data(file_path)
     trx_df = process_trx_with_bts_name(file_path, df_bts=df_bts)
     trx_summary_df = process_trx_data(file_path, trx_bts_name=trx_df)
     mal_summary_df = process_mal_data(file_path)
     gsm_df = process_gsm_data(file_path, df_trx=trx_summary_df, df_mal=mal_summary_df)
+    remember_processed(file_path, "gsm_data", gsm_df)
     mal_df = process_mal_with_bts_name(file_path, mal_df=mal_summary_df, df_bts=df_bts)
+    return [gsm_df, mal_df, trx_df]
+
+
+def combined_gsm_database(file_path: str):
+    gsm_df, mal_df, trx_df = get_or_build_processed(
+        file_path,
+        "gsm_bundle",
+        lambda: _build_combined_gsm_database(file_path),
+    )
 
     UtilsVars.all_db_dfs.extend([gsm_df, mal_df, trx_df])
     UtilsVars.gsm_dfs.extend([gsm_df, mal_df, trx_df])

queries/process_invunit.py

@@ -3,6 +3,7 @@ import pandas as pd
 from utils.convert_to_excel import convert_invunit_dfs, save_dataframe
 from utils.dump_excel import read_dump_excel
 from utils.extract_code import extract_code_from_mrbts
+from utils.processing_cache import get_or_build_processed
 from utils.utils_vars import UtilsVars
 
 RF_UNIT = [
@@ -112,7 +113,7 @@ def create_invunit_summary(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
-def build_invunit_number_dataframe(file_path: str) -> pd.DataFrame:
+def _build_invunit_number_dataframe(file_path: str) -> pd.DataFrame:
     """
     Build detailed INVUNIT_NUMBER dataframe from dump INVUNIT sheet.
     """
@@ -189,6 +190,17 @@ def build_invunit_number_dataframe(file_path: str) -> pd.DataFrame:
     return df_invunit_number
 
 
+def build_invunit_number_dataframe(file_path: str) -> pd.DataFrame:
+    """
+    Build detailed INVUNIT_NUMBER dataframe from dump INVUNIT sheet.
+    """
+    return get_or_build_processed(
+        file_path,
+        "invunit_number",
+        lambda: _build_invunit_number_dataframe(file_path),
+    )
+
+
 def process_invunit_number_data(file_path: str) -> pd.DataFrame:
     """
     Process and append INVUNIT_NUMBER dataframe to All DB buffers.
@@ -199,7 +211,7 @@ def process_invunit_number_data(file_path: str) -> pd.DataFrame:
     return df_invunit_number
 
 
-def process_invunit_data(file_path: str) -> pd.DataFrame:
+def _build_invunit_data(file_path: str) -> pd.DataFrame:
     """
     Process data from the specified file path.
 
@@ -251,6 +263,18 @@ def process_invunit_data(file_path: str) -> pd.DataFrame:
         ).tolist()
     ]
 
+    return df_invunit
+
+
+def process_invunit_data(file_path: str) -> pd.DataFrame:
+    """
+    Process data from the specified file path.
+    """
+    df_invunit = get_or_build_processed(
+        file_path,
+        "invunit",
+        lambda: _build_invunit_data(file_path),
+    )
     UtilsVars.all_db_dfs.append(df_invunit)
     UtilsVars.all_db_dfs_names.append("INVUNIT")
     return df_invunit

queries/process_lte.py

@@ -5,6 +5,7 @@ from utils.config_band import config_band, lte_mrbts_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.dump_excel import read_dump_excel
 from utils.kml_creator import generate_kml_from_df
+from utils.processing_cache import get_or_build_processed
 from utils.utils_vars import (
     LteFddAnalysisData,
     LteTddAnalysisData,
@@ -156,7 +157,7 @@ def process_lncel(file_path: str):
     return df_lncel
 
 
-def process_lte_data(file_path: str):
+def _build_lte_data(file_path: str) -> list[pd.DataFrame]:
     """
     Process data from the specified file path.
 
@@ -237,16 +238,27 @@ def process_lte_data(file_path: str):
     # Save dataframes
     # save_dataframe(df_fdd_final, "fdd")
     # save_dataframe(df_tdd_final, "tdd")
-    UtilsVars.all_db_dfs.extend([df_fdd_final, df_tdd_final])
-    UtilsVars.lte_dfs.extend([df_fdd_final, df_tdd_final])
-    UtilsVars.all_db_dfs_names.extend(["LTE_FDD", "LTE_TDD"])
-
     return [df_fdd_final, df_tdd_final]
     # add the fdd and tdd to the list
 
     # UtilsVars.final_lte_database = [df_fdd_final, df_tdd_final]
 
 
+def process_lte_data(file_path: str):
+    """
+    Process LTE data and preserve the historical UtilsVars side effects.
+    """
+    lte_dfs = get_or_build_processed(
+        file_path,
+        "lte_data",
+        lambda: _build_lte_data(file_path),
+    )
+    UtilsVars.all_db_dfs.extend(lte_dfs)
+    UtilsVars.lte_dfs.extend(lte_dfs)
+    UtilsVars.all_db_dfs_names.extend(["LTE_FDD", "LTE_TDD"])
+    return lte_dfs
+
+
 def process_lte_data_to_excel(file_path: str):
     lte_dfs = process_lte_data(file_path)
     UtilsVars.final_lte_database = convert_dfs(lte_dfs, ["LTE_FDD", "LTE_TDD"])

queries/process_trx.py

@@ -141,24 +141,12 @@ def process_trx_with_bts_name(
     # TCHs	SDs	BCCH	CCCH	CBC	Total	Signal
 
     # Calculate "count of channels per TRX" for each row
-    df_trx_bts_name["TCHs"] = df_trx_bts_name[channel_columns].apply(
-        lambda row: (row == 2).sum(), axis=1
-    )
-    df_trx_bts_name["SDs"] = df_trx_bts_name[channel_columns].apply(
-        lambda row: (row == 3).sum(), axis=1
-    )
-
-    df_trx_bts_name["BCCHs"] = df_trx_bts_name[channel_columns].apply(
-        lambda row: (row == 4).sum(), axis=1
-    )
-
-    df_trx_bts_name["CCCHs"] = df_trx_bts_name[channel_columns].apply(
-        lambda row: (row == 6).sum(), axis=1
-    )
-
-    df_trx_bts_name["CBCs"] = df_trx_bts_name[channel_columns].apply(
-        lambda row: (row == 8).sum(), axis=1
-    )
+    channel_data = df_trx_bts_name[channel_columns]
+    df_trx_bts_name["TCHs"] = channel_data.eq(2).sum(axis=1)
+    df_trx_bts_name["SDs"] = channel_data.eq(3).sum(axis=1)
+    df_trx_bts_name["BCCHs"] = channel_data.eq(4).sum(axis=1)
+    df_trx_bts_name["CCCHs"] = channel_data.eq(6).sum(axis=1)
+    df_trx_bts_name["CBCs"] = channel_data.eq(8).sum(axis=1)
 
     # Total Channels =  TCHs + SDs + BCCHs + CCCHs + CBCs
 
@@ -175,27 +163,20 @@ def process_trx_with_bts_name(
     df_trx_bts_name["Signal"] = (
         df_trx_bts_name["BCCHs"] + df_trx_bts_name["CCCHs"] + df_trx_bts_name["CBCs"]
     )
-    df_trx_bts_name["number_tch_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "TCHs"
-    ].transform("sum")
-    df_trx_bts_name["number_sd_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "SDs"
-    ].transform("sum")
-    df_trx_bts_name["number_bcch_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "BCCHs"
-    ].transform("sum")
-    df_trx_bts_name["number_ccch_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "CCCHs"
-    ].transform("sum")
-    df_trx_bts_name["number_cbc_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "CBCs"
-    ].transform("sum")
-    df_trx_bts_name["number_total_channels_per_cell"] = df_trx_bts_name.groupby(
-        "ID_BTS"
-    )["TotalChannels"].transform("sum")
-    df_trx_bts_name["number_signals_per_cell"] = df_trx_bts_name.groupby("ID_BTS")[
-        "Signal"
+    channel_total_columns = {
+        "TCHs": "number_tch_per_cell",
+        "SDs": "number_sd_per_cell",
+        "BCCHs": "number_bcch_per_cell",
+        "CCCHs": "number_ccch_per_cell",
+        "CBCs": "number_cbc_per_cell",
+        "TotalChannels": "number_total_channels_per_cell",
+        "Signal": "number_signals_per_cell",
+    }
+    totals_per_cell = df_trx_bts_name.groupby("ID_BTS")[
+        list(channel_total_columns)
     ].transform("sum")
+    totals_per_cell.rename(columns=channel_total_columns, inplace=True)
+    df_trx_bts_name = pd.concat([df_trx_bts_name, totals_per_cell], axis=1)
 
     # Avoir les TRX par bande et par secteur et BCF sous forme concaténée comme 5/3/3
 

queries/process_wcdma.py

@@ -5,6 +5,7 @@ from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.dump_excel import read_dump_excel
 from utils.extract_code import extract_code_from_mrbts
 from utils.kml_creator import generate_kml_from_df
+from utils.processing_cache import get_or_build_processed
 from utils.utils_vars import UtilsVars, WcdmaAnalysisData, get_physical_db
 
 WCEL_COLUMNS = [
@@ -94,7 +95,7 @@ WCDMA_KML_COLUMNS = [
 ]
 
 
-def process_wcdma_data(file_path: str):
+def _build_wcdma_data(file_path: str) -> pd.DataFrame:
     """
     Process data from the specified file path.
 
@@ -190,17 +191,27 @@ def process_wcdma_data(file_path: str):
     # save_dataframe(df_wcel_bcf, "wbts")
     # save_dataframe(df_wncel, "wncel")
     # df_3g = save_dataframe(df_3g, "3G")
-    UtilsVars.all_db_dfs.append(df_3g)
-    UtilsVars.wcdma_dfs.append(df_3g)
-    UtilsVars.all_db_dfs_names.append("WCDMA")
-
-    # UtilsVars.final_wcdma_database = convert_dfs([df_3g], ["WCDMA"])
     return df_3g
     # UtilsVars.final_wcdma_database = [df_3g]
 
     # BTS.process_ok = "Done"
 
 
+def process_wcdma_data(file_path: str):
+    """
+    Process WCDMA data and preserve the historical UtilsVars side effects.
+    """
+    df_3g = get_or_build_processed(
+        file_path,
+        "wcdma_data",
+        lambda: _build_wcdma_data(file_path),
+    )
+    UtilsVars.all_db_dfs.append(df_3g)
+    UtilsVars.wcdma_dfs.append(df_3g)
+    UtilsVars.all_db_dfs_names.append("WCDMA")
+    return df_3g
+
+
 def process_wcdma_data_to_excel(file_path: str):
     """
     Process WCDMA data from the specified file path and convert it to Excel format

scripts/benchmark_database_processing.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import argparse
+import gc
+import sys
+import time
+import tracemalloc
+from collections.abc import Callable
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from queries.process_all_db import clear_all_dbs, process_all_tech_db, process_atoll_db, process_nice_db
+from queries.process_gsm import process_gsm_data_to_excel
+from queries.process_lte import process_lte_data_to_excel
+from utils.dump_excel import clear_dump_excel_cache
+from utils.processing_cache import clear_processed_dataframe_cache
+from utils.utils_vars import UtilsVars
+
+
+def reset_outputs(*, clear_caches: bool) -> None:
+    clear_all_dbs()
+    UtilsVars.final_gsm_database = ""
+    UtilsVars.final_lte_database = ""
+    UtilsVars.final_nice_database = None
+    UtilsVars.final_atoll_database = None
+    if clear_caches:
+        clear_dump_excel_cache()
+        clear_processed_dataframe_cache()
+    gc.collect()
+
+
+def run_case(
+    label: str,
+    func: Callable[[str], None],
+    dump_path: str,
+    *,
+    clear_caches: bool,
+    measure_memory: bool,
+) -> dict[str, float | str]:
+    reset_outputs(clear_caches=clear_caches)
+    if measure_memory:
+        tracemalloc.start()
+    start = time.perf_counter()
+    func(dump_path)
+    elapsed = time.perf_counter() - start
+    peak_mib = 0.0
+    if measure_memory:
+        _, peak = tracemalloc.get_traced_memory()
+        tracemalloc.stop()
+        peak_mib = peak / 1024 / 1024
+    return {"case": label, "seconds": elapsed, "peak_mib": peak_mib}
+
+
+def print_result(result: dict[str, float | str], *, measure_memory: bool) -> None:
+    memory = f" peak={result['peak_mib']:.1f} MiB" if measure_memory else ""
+    print(f"{result['case']}: {result['seconds']:.2f}s{memory}", flush=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Benchmark OML_DB database generation paths.")
+    parser.add_argument("dump", type=Path, help="Path to a dump .xlsb file.")
+    parser.add_argument(
+        "--memory",
+        action="store_true",
+        help="Measure approximate Python allocation peak with tracemalloc. Slower.",
+    )
+    args = parser.parse_args()
+    dump_path = str(args.dump.expanduser().resolve())
+
+    cases: list[tuple[str, Callable[[str], None]]] = [
+        ("Generate 2G DB", process_gsm_data_to_excel),
+        ("Generate LTE DB", process_lte_data_to_excel),
+        ("Generate All DBs", process_all_tech_db),
+        ("Generate Nice DB", process_nice_db),
+        ("Generate Atoll DB", process_atoll_db),
+    ]
+
+    for label, func in cases:
+        print_result(
+            run_case(label, func, dump_path, clear_caches=True, measure_memory=args.memory),
+            measure_memory=args.memory,
+        )
+
+    print_result(
+        run_case(
+            "Generate All DBs repeat #1",
+            process_all_tech_db,
+            dump_path,
+            clear_caches=True,
+            measure_memory=args.memory,
+        ),
+        measure_memory=args.memory,
+    )
+    print_result(
+        run_case(
+            "Generate All DBs repeat #2 same process",
+            process_all_tech_db,
+            dump_path,
+            clear_caches=False,
+            measure_memory=args.memory,
+        ),
+        measure_memory=args.memory,
+    )
+
+
+if __name__ == "__main__":
+    main()

tests/test_processing_reuse.py

@@ -0,0 +1,180 @@
+import pandas as pd
+
+from utils.processing_cache import clear_processed_dataframe_cache
+from utils.utils_vars import UtilsVars
+
+
+def setup_function():
+    clear_processed_dataframe_cache()
+    UtilsVars.all_db_dfs = []
+    UtilsVars.all_db_dfs_names = []
+    UtilsVars.gsm_dfs = []
+    UtilsVars.wcdma_dfs = []
+    UtilsVars.lte_dfs = []
+
+
+def assert_same_frame(left: pd.DataFrame, right: pd.DataFrame) -> None:
+    pd.testing.assert_frame_equal(
+        left.reset_index(drop=True),
+        right.reset_index(drop=True),
+        check_dtype=True,
+        check_like=False,
+    )
+
+
+def test_lte_processed_cache_preserves_frames_and_side_effects(monkeypatch):
+    from queries import process_lte
+
+    calls = {"count": 0}
+    expected_fdd = pd.DataFrame({"A": pd.Series([1, None], dtype="Int64"), "B": ["x", None]})
+    expected_tdd = pd.DataFrame({"C": pd.Series([1.5, None], dtype="Float64"), "D": ["y", None]})
+
+    def fake_build(file_path):
+        calls["count"] += 1
+        return [expected_fdd.copy(deep=True), expected_tdd.copy(deep=True)]
+
+    monkeypatch.setattr(process_lte, "_build_lte_data", fake_build)
+
+    first_fdd, first_tdd = process_lte.process_lte_data("dump.xlsb")
+    first_fdd.loc[0, "B"] = "mutated"
+
+    second_fdd, second_tdd = process_lte.process_lte_data("dump.xlsb")
+
+    assert calls["count"] == 1
+    assert_same_frame(second_fdd, expected_fdd)
+    assert_same_frame(first_tdd, expected_tdd)
+    assert_same_frame(second_tdd, expected_tdd)
+    assert UtilsVars.all_db_dfs_names == ["LTE_FDD", "LTE_TDD", "LTE_FDD", "LTE_TDD"]
+
+
+def test_wcdma_processed_cache_preserves_frame_and_side_effects(monkeypatch):
+    from queries import process_wcdma
+
+    calls = {"count": 0}
+    expected = pd.DataFrame(
+        {
+            "ID_WCEL": pd.Series(["1_2_3", None], dtype="string"),
+            "LAC": pd.Series([10, None], dtype="Int64"),
+        }
+    )
+
+    def fake_build(file_path):
+        calls["count"] += 1
+        return expected.copy(deep=True)
+
+    monkeypatch.setattr(process_wcdma, "_build_wcdma_data", fake_build)
+
+    first = process_wcdma.process_wcdma_data("dump.xlsb")
+    first.loc[0, "ID_WCEL"] = "mutated"
+    second = process_wcdma.process_wcdma_data("dump.xlsb")
+
+    assert calls["count"] == 1
+    assert_same_frame(second, expected)
+    assert UtilsVars.all_db_dfs_names == ["WCDMA", "WCDMA"]
+
+
+def test_gsm_bundle_cache_preserves_gsm_mal_trx(monkeypatch):
+    from queries import process_gsm
+
+    calls = {"count": 0}
+    gsm = pd.DataFrame({"ID_BTS": pd.Series(["1_2_3"], dtype="string"), "BCCH": pd.Series([10], dtype="Int64")})
+    mal = pd.DataFrame({"ID_MAL": pd.Series(["1_4"], dtype="string"), "MAL_TCH": ["1,2"]})
+    trx = pd.DataFrame({"TRX": pd.Series([1], dtype="Int64"), "name": ["BTS_A"]})
+
+    def fake_build(file_path):
+        calls["count"] += 1
+        return [gsm.copy(deep=True), mal.copy(deep=True), trx.copy(deep=True)]
+
+    monkeypatch.setattr(process_gsm, "_build_combined_gsm_database", fake_build)
+
+    first = process_gsm.combined_gsm_database("dump.xlsb")
+    first[0].loc[0, "ID_BTS"] = "mutated"
+    second = process_gsm.combined_gsm_database("dump.xlsb")
+
+    assert calls["count"] == 1
+    for actual, expected in zip(second, [gsm, mal, trx]):
+        assert_same_frame(actual, expected)
+    assert UtilsVars.all_db_dfs_names == ["GSM", "MAL", "TRX", "GSM", "MAL", "TRX"]
+
+
+def test_invunit_caches_preserve_nulls_columns_and_dtypes(monkeypatch):
+    from queries import process_invunit
+
+    invunit_calls = {"count": 0}
+    number_calls = {"count": 0}
+    invunit = pd.DataFrame(
+        {
+            "MRBTS": pd.Series(["12345", None], dtype="string"),
+            "code": pd.Series([123, None], dtype="Int64"),
+            "invunit_summary": pd.Series(["1 FBBA", None], dtype="string"),
+        }
+    )
+    invunit_number = pd.DataFrame(
+        {
+            "MRBTS": pd.Series(["12345", None], dtype="string"),
+            "name": pd.Series(["SITE_A", None], dtype="string"),
+            "inventoryUnitType": pd.Series(["FBBA", None], dtype="string"),
+            "vendorUnitTypeNumber": pd.Series(["V1", None], dtype="string"),
+            "serialNumber": pd.Series(["S1", None], dtype="string"),
+        }
+    )
+
+    def fake_invunit(file_path):
+        invunit_calls["count"] += 1
+        return invunit.copy(deep=True)
+
+    def fake_invunit_number(file_path):
+        number_calls["count"] += 1
+        return invunit_number.copy(deep=True)
+
+    monkeypatch.setattr(process_invunit, "_build_invunit_data", fake_invunit)
+    monkeypatch.setattr(process_invunit, "_build_invunit_number_dataframe", fake_invunit_number)
+
+    first = process_invunit.process_invunit_data("dump.xlsb")
+    first.loc[0, "invunit_summary"] = "mutated"
+    second = process_invunit.process_invunit_data("dump.xlsb")
+    number_first = process_invunit.build_invunit_number_dataframe("dump.xlsb")
+    number_first.loc[0, "serialNumber"] = "mutated"
+    number_second = process_invunit.build_invunit_number_dataframe("dump.xlsb")
+
+    assert invunit_calls["count"] == 1
+    assert number_calls["count"] == 1
+    assert_same_frame(second, invunit)
+    assert_same_frame(number_second, invunit_number)
+
+
+def test_physical_db_cache_invalidates_when_file_changes(tmp_path, monkeypatch):
+    import utils.utils_vars as utils_vars
+
+    physical_path = tmp_path / "physical_database.csv"
+    columns = [
+        "Code_Sector",
+        "Azimut",
+        "Longitude",
+        "Latitude",
+        "Hauteur",
+        "City",
+        "Adresse",
+        "Commune",
+        "Cercle",
+    ]
+    pd.DataFrame([["1_1", 10, -1.0, 2.0, 30, "A", "Addr", "Com", "Cer"]], columns=columns).to_csv(
+        physical_path,
+        index=False,
+    )
+    monkeypatch.setattr(utils_vars, "url", str(physical_path))
+    monkeypatch.setattr(utils_vars, "_PHYSICAL_DB_CACHE", None)
+
+    first = utils_vars.get_physical_db()
+    first.loc[0, "City"] = "mutated"
+    second = utils_vars.get_physical_db()
+    assert second.loc[0, "City"] == "A"
+
+    pd.DataFrame([["1_1", 20, -1.0, 2.0, 30, "B_LONG", "Addr", "Com", "Cer"]], columns=columns).to_csv(
+        physical_path,
+        index=False,
+    )
+
+    third = utils_vars.get_physical_db()
+    assert third.loc[0, "Azimut"] == 20
+    assert third.loc[0, "City"] == "B_LONG"

utils/dump_excel.py

@@ -67,6 +67,13 @@ def _file_cache_key(file_path) -> tuple:
     return ("object", id(file_path), name, size)
 
 
+def get_dump_file_cache_key(file_path) -> tuple:
+    """
+    Return the cache identity used for a dump path/upload object.
+    """
+    return _file_cache_key(file_path)
+
+
 def _freeze_kwargs(kwargs: dict) -> tuple:
     return tuple(sorted((key, repr(value)) for key, value in kwargs.items()))
 

utils/processing_cache.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+from collections.abc import Callable
+from typing import TypeVar
+
+import pandas as pd
+
+from utils.dump_excel import get_dump_file_cache_key
+from utils.utils_vars import UtilsVars
+
+
+T = TypeVar("T")
+
+_MAX_PROCESSED_CACHE_ITEMS = 16
+_PROCESSED_CACHE: OrderedDict[tuple, object] = OrderedDict()
+
+
+def _clone(value):
+    if isinstance(value, pd.DataFrame):
+        return value.copy(deep=True)
+    if isinstance(value, list):
+        return [_clone(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(_clone(item) for item in value)
+    if isinstance(value, dict):
+        return {key: _clone(item) for key, item in value.items()}
+    return value
+
+
+def _remember(key: tuple, value) -> None:
+    _PROCESSED_CACHE[key] = _clone(value)
+    _PROCESSED_CACHE.move_to_end(key)
+    while len(_PROCESSED_CACHE) > _MAX_PROCESSED_CACHE_ITEMS:
+        _PROCESSED_CACHE.popitem(last=False)
+
+
+def _cache_key(file_path, namespace: str, extra: tuple = ()) -> tuple:
+    return (
+        namespace,
+        get_dump_file_cache_key(file_path),
+        bool(UtilsVars.exclude_decommissioned_2g_bsc),
+        tuple(sorted(UtilsVars.decommissioned_2g_bsc_ids)),
+        extra,
+    )
+
+
+def clear_processed_dataframe_cache() -> None:
+    _PROCESSED_CACHE.clear()
+
+
+def get_or_build_processed(
+    file_path,
+    namespace: str,
+    builder: Callable[[], T],
+    *,
+    extra: tuple = (),
+) -> T:
+    key = _cache_key(file_path, namespace, extra)
+    if key in _PROCESSED_CACHE:
+        _PROCESSED_CACHE.move_to_end(key)
+        return _clone(_PROCESSED_CACHE[key])
+
+    value = builder()
+    _remember(key, value)
+    return _clone(value)
+
+
+def remember_processed(
+    file_path,
+    namespace: str,
+    value: T,
+    *,
+    extra: tuple = (),
+) -> None:
+    _remember(_cache_key(file_path, namespace, extra), value)

utils/utils_vars.py

@@ -1,8 +1,10 @@
 import numpy as np
 import pandas as pd
+from pathlib import Path
 
 # url = "https://raw.githubusercontent.com/DavMelchi/STORAGE/refs/heads/main/physical_db/physical_database.csv"
 url = r"./physical_db/physical_database.csv"
+_PHYSICAL_DB_CACHE: tuple[tuple[str, int, int], pd.DataFrame] | None = None
 
 
 def get_physical_db():
@@ -14,7 +16,14 @@ def get_physical_db():
     Returns:
         pd.DataFrame: A DataFrame containing the filtered columns.
     """
-    physical = pd.read_csv(url)
+    global _PHYSICAL_DB_CACHE
+    path = Path(url)
+    stat = path.stat()
+    cache_key = (str(path.resolve()), stat.st_size, stat.st_mtime_ns)
+    if _PHYSICAL_DB_CACHE is not None and _PHYSICAL_DB_CACHE[0] == cache_key:
+        return _PHYSICAL_DB_CACHE[1].copy(deep=True)
+
+    physical = pd.read_csv(path)
     physical = physical[
         [
             "Code_Sector",
@@ -28,7 +37,8 @@ def get_physical_db():
             "Cercle",
         ]
     ]
-    return physical
+    _PHYSICAL_DB_CACHE = (cache_key, physical)
+    return physical.copy(deep=True)
 
 
 class UtilsVars: