Reduce history wildcard payloads and add runtime virtual-interface prefix filtering

pm_dashboard/data_logger.py

@@ -25,6 +25,8 @@
 
 class DataLogger:
 
+    DEFAULT_VIRTUAL_INTERFACE_PREFIXES = ['veth', 'br-', 'docker', 'virbr', 'vmnet']
+
     @log_error
     def __init__(self, database='pm_dashboard', interval=1, spc_enabled=False, get_logger=None):
         if get_logger is None:
@@ -45,6 +47,7 @@ def __init__(self, database='pm_dashboard', interval=1, spc_enabled=False, get_l
 
         self.db = Database(database, get_logger=get_logger)
         self.interval = interval
+        self.virtual_interface_prefixes = list(self.DEFAULT_VIRTUAL_INTERFACE_PREFIXES)
         if spc_enabled:
             self.log.info("SPC peripheral enabled")
             from spc.spc import SPC
@@ -68,11 +71,21 @@ def update_status(self, status):
     def set_interval(self, interval):
         self.interval = interval
 
+    @log_error
+    def set_virtual_interface_prefixes(self, prefixes):
+        if isinstance(prefixes, list) and all(isinstance(prefix, str) for prefix in prefixes):
+            self.virtual_interface_prefixes = prefixes
+
+    def _get_exclude_prefixes(self):
+        prefixes = tuple(self.virtual_interface_prefixes)
+        return prefixes or None
+
     @log_error
     def get_data(self):
+        exclude_prefixes = self._get_exclude_prefixes()
         boot_time = get_boot_time()
-        ips = get_ips()
-        macs = get_macs()
+        ips = get_ips(exclude_prefixes=exclude_prefixes)
+        macs = get_macs(exclude_prefixes=exclude_prefixes)
         network_connection_type = get_network_connection_type()
         network_speed = get_network_speed()
 
@@ -111,11 +124,9 @@ def get_data(self):
 
         data['boot_time'] = float(boot_time)
 
-        ips = get_ips()
         for name in ips:
             data[f'ip_{name}'] = ips[name]
 
-        macs = get_macs()
         for name in macs:
             data[f'mac_{name}'] = macs[name]
 

pm_dashboard/database.py

@@ -13,8 +13,46 @@ def __init__(self, database, get_logger=None):
         self.log = get_logger(__name__)
         self.database = database
         self.influx_manually_started = False
+        self._field_keys_cache = {}
+        self._virtual_prefixes = self._build_virtual_field_prefixes(['veth', 'br-', 'docker', 'virbr', 'vmnet'])
 
         self.client = InfluxDBClient(host='localhost', port=8086)
+
+    @staticmethod
+    def _build_virtual_field_prefixes(interface_prefixes):
+        return tuple(
+            f'{field_type}_{iface}'
+            for iface in interface_prefixes
+            for field_type in ('mac', 'ip')
+        )
+
+    @staticmethod
+    def _quote_identifier(identifier):
+        return '"' + identifier.replace('"', '\\"') + '"'
+
+    def set_virtual_prefixes(self, interface_prefixes):
+        if not isinstance(interface_prefixes, list) or not all(isinstance(prefix, str) for prefix in interface_prefixes):
+            return
+        self._virtual_prefixes = self._build_virtual_field_prefixes(interface_prefixes)
+        self._field_keys_cache.clear()
+
+    def get_stable_field_keys(self, measurement, max_age=60):
+        cached = self._field_keys_cache.get(measurement)
+        if cached and (time.time() - cached[0]) < max_age:
+            return cached[1]
+
+        try:
+            result = self.client.query(f'SHOW FIELD KEYS FROM "{measurement}"')
+            all_keys = [row['fieldKey'] for row in result.get_points()]
+            stable_keys = [
+                key for key in all_keys
+                if not key.startswith(self._virtual_prefixes)
+            ]
+            self._field_keys_cache[measurement] = (time.time(), stable_keys)
+            return stable_keys
+        except Exception as e:
+            self.log.warning(f"get_stable_field_keys failed for {measurement}: {e}")
+            return []
 
     def set_debug_level(self, level):
         self.log.info(f"Setting debug level to {level}")
@@ -113,7 +151,11 @@ def get_data_by_time_range(self, measurement, start_time, end_time, keys="*", fu
         if function not in ["mean", "sum", "min", "max", "count"]:
             self.log.error(f"Invalid function: {function}")
             return []
-        if keys != "*":
+        if keys == "*":
+            stable_keys = self.get_stable_field_keys(measurement)
+            if stable_keys:
+                keys = ",".join(self._quote_identifier(k) for k in stable_keys)
+        else:
             newKeys = []
             for k in keys.split(","):
                 newKeys.append(f'{function}("{k}") as "{k}"')
@@ -142,6 +184,11 @@ def get(self, measurement, key="*", n=1):
         if not self.is_ready():
             self.log.error('Database is not ready')
             return []
+        if key == "*":
+            stable_keys = self.get_stable_field_keys(measurement)
+            if stable_keys:
+                key = ",".join(self._quote_identifier(k) for k in stable_keys)
+
         for _ in range(3):
             query = f"SELECT {key} FROM {measurement} ORDER BY time DESC LIMIT {n}"
             result = self.client.query(query)

pm_dashboard/pm_dashboard.py

@@ -39,6 +39,7 @@
 __device_info__ = {}
 __mqtt_connected__ = False
 __enable_history__ = False
+DEFAULT_VIRTUAL_INTERFACE_PREFIXES = ['veth', 'br-', 'docker', 'virbr', 'vmnet']
 
 __on_outside_config_changed__ = lambda config: None
 __on_inside_config_changed__ = lambda config: None
@@ -283,9 +284,19 @@ def get_disk_list():
 @__app__.route(f'{__api_prefix__}/get-network-interface-list')
 @cross_origin()
 def get_network_interface_list():
-    interfaces = list(get_ips().keys())
+    prefixes = tuple(__config__.get('system', {}).get('virtual_interface_prefixes', DEFAULT_VIRTUAL_INTERFACE_PREFIXES))
+    interfaces = list(get_ips(exclude_prefixes=prefixes or None).keys())
     return {"status": True, "data": interfaces}
 
+@__app__.route(f'{__api_prefix__}/set-virtual-interface-prefixes', methods=['POST'])
+@cross_origin()
+def set_virtual_interface_prefixes():
+    prefixes = request.json.get("prefixes")
+    if not isinstance(prefixes, list) or not all(isinstance(prefix, str) for prefix in prefixes):
+        return {"status": False, "error": "[ERROR] prefixes must be a list of strings"}
+    __on_config_changed__({'system': {'virtual_interface_prefixes': prefixes}})
+    return {"status": True, "data": "OK"}
+
 @__app__.route(f'{__api_prefix__}/set-temperature-unit', methods=['POST'])
 @cross_origin()
 def set_temperature_unit():
@@ -479,16 +490,20 @@ def __init__(self, device_info=None, database='pm_dashboard', spc_enabled=False,
         __config__ = config
         if 'enable_history' not in __config__['system']:
             __config__['system']['enable_history'] = False
+        if 'virtual_interface_prefixes' not in __config__['system']:
+            __config__['system']['virtual_interface_prefixes'] = list(DEFAULT_VIRTUAL_INTERFACE_PREFIXES)
         __enable_history__ = config['system']['enable_history']
 
         self.data_logger = DataLogger(
             database=database,
             spc_enabled=spc_enabled,
             interval=__config__['system']['data_interval'],
             get_logger=get_logger)
+        self.data_logger.set_virtual_interface_prefixes(__config__['system']['virtual_interface_prefixes'])
         __data_logger__ = self.data_logger
         if __enable_history__:
             __db__ = Database(database, get_logger=get_logger)
+            __db__.set_virtual_prefixes(__config__['system']['virtual_interface_prefixes'])
 
         self.started = False
         __on_inside_config_changed__ = self.on_config_changed
@@ -516,8 +531,14 @@ def start(self):
 
     @log_error
     def on_config_changed(self, config):
+        global __enable_history__, __db__
         if 'data_interval' in config['system']:
             self.data_logger.set_interval(config['system']['data_interval'])
+        if 'virtual_interface_prefixes' in config['system']:
+            prefixes = config['system']['virtual_interface_prefixes']
+            self.data_logger.set_virtual_interface_prefixes(prefixes)
+            if __db__ is not None:
+                __db__.set_virtual_prefixes(prefixes)
         if 'enable_history' in config['system']:
             if config['system']['enable_history'] == True:
                 if __enable_history__ == False: