[ci] Merge components with different buses to reduce CI time

script/analyze_component_buses.py

@@ -366,6 +366,65 @@ def analyze_all_components(
     return components, non_groupable, direct_bus_components
 
 
+@lru_cache(maxsize=256)
+def _get_bus_configs(buses: tuple[str, ...]) -> frozenset[tuple[str, str]]:
+    """Map bus type to set of configs for that type.
+
+    Args:
+        buses: Tuple of bus package names (e.g., ("uart_9600", "i2c"))
+
+    Returns:
+        Frozenset of (base_type, full_config) tuples
+        Example: frozenset({("uart", "uart_9600"), ("i2c", "i2c")})
+    """
+    # Split on underscore to get base type: "uart_9600" -> "uart", "i2c" -> "i2c"
+    return frozenset((bus.split("_", 1)[0], bus) for bus in buses)
+
+
+@lru_cache(maxsize=1024)
+def are_buses_compatible(buses1: tuple[str, ...], buses2: tuple[str, ...]) -> bool:
+    """Check if two bus tuples are compatible for merging.
+
+    Two bus lists are compatible if they don't have conflicting configurations
+    for the same bus type. For example:
+    - ("ble", "uart") and ("i2c",) are compatible (different buses)
+    - ("uart_9600",) and ("uart_19200",) are NOT compatible (same bus, different configs)
+    - ("uart_9600",) and ("uart_9600",) are compatible (same bus, same config)
+
+    Args:
+        buses1: First tuple of bus package names
+        buses2: Second tuple of bus package names
+
+    Returns:
+        True if buses can be merged without conflicts
+    """
+    configs1 = _get_bus_configs(buses1)
+    configs2 = _get_bus_configs(buses2)
+
+    # Group configs by base type
+    bus_types1: dict[str, set[str]] = {}
+    for base_type, full_config in configs1:
+        if base_type not in bus_types1:
+            bus_types1[base_type] = set()
+        bus_types1[base_type].add(full_config)
+
+    bus_types2: dict[str, set[str]] = {}
+    for base_type, full_config in configs2:
+        if base_type not in bus_types2:
+            bus_types2[base_type] = set()
+        bus_types2[base_type].add(full_config)
+
+    # Check for conflicts: same bus type with different configs
+    for bus_type, configs in bus_types1.items():
+        if bus_type not in bus_types2:
+            continue  # No conflict - different bus types
+        # Same bus type - check if configs match
+        if configs != bus_types2[bus_type]:
+            return False  # Conflict - same bus type, different configs
+
+    return True  # No conflicts found
+
+
 def create_grouping_signature(
     platform_buses: dict[str, list[str]], platform: str
 ) -> str:

script/merge_component_configs.py

@@ -16,6 +16,7 @@ The merger handles:
 from __future__ import annotations
 
 import argparse
+from functools import lru_cache
 from pathlib import Path
 import re
 import sys
@@ -28,6 +29,10 @@ from esphome import yaml_util
 from esphome.config_helpers import merge_config
 from script.analyze_component_buses import PACKAGE_DEPENDENCIES, get_common_bus_packages
 
+# Prefix for dependency markers in package tracking
+# Used to mark packages that are included transitively (e.g., uart via modbus)
+DEPENDENCY_MARKER_PREFIX = "_dep_"
+
 
 def load_yaml_file(yaml_file: Path) -> dict:
     """Load YAML file using ESPHome's YAML loader.
@@ -44,6 +49,34 @@ def load_yaml_file(yaml_file: Path) -> dict:
     return yaml_util.load_yaml(yaml_file)
 
 
+@lru_cache(maxsize=256)
+def get_component_packages(
+    component_name: str, platform: str, tests_dir_str: str
+) -> dict:
+    """Get packages dict from a component's test file with caching.
+
+    This function is cached to avoid re-loading and re-parsing the same file
+    multiple times when extracting packages during cross-bus merging.
+
+    Args:
+        component_name: Name of the component
+        platform: Platform name (e.g., "esp32-idf")
+        tests_dir_str: String path to tests/components directory (must be string for cache hashability)
+
+    Returns:
+        Dictionary with 'packages' key containing the raw packages dict from the YAML,
+        or empty dict if no packages section exists
+    """
+    tests_dir = Path(tests_dir_str)
+    test_file = tests_dir / component_name / f"test.{platform}.yaml"
+    comp_data = load_yaml_file(test_file)
+
+    if "packages" not in comp_data or not isinstance(comp_data["packages"], dict):
+        return {}
+
+    return comp_data["packages"]
+
+
 def extract_packages_from_yaml(data: dict) -> dict[str, str]:
     """Extract COMMON BUS package includes from parsed YAML.
 
@@ -82,7 +115,7 @@ def extract_packages_from_yaml(data: dict) -> dict[str, str]:
             if dep not in common_bus_packages:
                 continue
             # Mark as included via dependency
-            packages[f"_dep_{dep}"] = f"(included via {name})"
+            packages[f"{DEPENDENCY_MARKER_PREFIX}{dep}"] = f"(included via {name})"
 
     return packages
 
@@ -195,6 +228,9 @@ def merge_component_configs(
     # Start with empty config
     merged_config_data = {}
 
+    # Convert tests_dir to string for caching
+    tests_dir_str = str(tests_dir)
+
     # Process each component
     for comp_name in component_names:
         comp_dir = tests_dir / comp_name
@@ -206,26 +242,29 @@ def merge_component_configs(
         # Load the component's test file
         comp_data = load_yaml_file(test_file)
 
-        # Validate packages are compatible
-        # Components with no packages (no_buses) can merge with any group
+        # Merge packages from all components (cross-bus merging)
+        # Components can have different packages (e.g., one with ble, another with uart)
+        # as long as they don't conflict (checked by are_buses_compatible before calling this)
         comp_packages = extract_packages_from_yaml(comp_data)
 
         if all_packages is None:
-            # First component - set the baseline
-            all_packages = comp_packages
-        elif not comp_packages:
-            # This component has no packages (no_buses) - it can merge with any group
-            pass
-        elif not all_packages:
-            # Previous components had no packages, but this one does - adopt these packages
-            all_packages = comp_packages
-        elif comp_packages != all_packages:
-            # Both have packages but they differ - this is an error
-            raise ValueError(
-                f"Component {comp_name} has different packages than previous components. "
-                f"Expected: {all_packages}, Got: {comp_packages}. "
-                f"All components must use the same common bus configs to be merged."
-            )
+            # First component - initialize package dict
+            all_packages = comp_packages if comp_packages else {}
+        elif comp_packages:
+            # Merge packages - combine all unique package types
+            # If both have the same package type, verify they're identical
+            for pkg_name, pkg_config in comp_packages.items():
+                if pkg_name in all_packages:
+                    # Same package type - verify config matches
+                    if all_packages[pkg_name] != pkg_config:
+                        raise ValueError(
+                            f"Component {comp_name} has conflicting config for package '{pkg_name}'. "
+                            f"Expected: {all_packages[pkg_name]}, Got: {pkg_config}. "
+                            f"Components with conflicting bus configs cannot be merged."
+                        )
+                else:
+                    # New package type - add it
+                    all_packages[pkg_name] = pkg_config
 
         # Handle $component_dir by replacing with absolute path
         # This allows components that use local file references to be grouped
@@ -287,26 +326,38 @@ def merge_component_configs(
         # merge_config handles list merging with ID-based deduplication automatically
         merged_config_data = merge_config(merged_config_data, comp_data)
 
-    # Add packages back (only once, since they're identical)
-    # IMPORTANT: Only re-add common bus packages (spi, i2c, uart, etc.)
+    # Add merged packages back (union of all component packages)
+    # IMPORTANT: Only include common bus packages (spi, i2c, uart, etc.)
     # Do NOT re-add component-specific packages as they contain unprefixed $component_dir refs
     if all_packages:
-        first_comp_data = load_yaml_file(
-            tests_dir / component_names[0] / f"test.{platform}.yaml"
-        )
-        if "packages" in first_comp_data and isinstance(
-            first_comp_data["packages"], dict
-        ):
-            # Filter to only include common bus packages
-            # Only dict format can contain common bus packages
-            common_bus_packages = get_common_bus_packages()
-            filtered_packages = {
-                name: value
-                for name, value in first_comp_data["packages"].items()
-                if name in common_bus_packages
-            }
-            if filtered_packages:
-                merged_config_data["packages"] = filtered_packages
+        # Build packages dict from merged all_packages
+        # all_packages is a dict mapping package_name -> str(package_value)
+        # We need to reconstruct the actual package values by loading them from any component
+        # Since packages with the same name must have identical configs (verified above),
+        # we can load the package value from the first component that has each package
+        common_bus_packages = get_common_bus_packages()
+        merged_packages: dict[str, Any] = {}
+
+        for pkg_name in all_packages:
+            # Skip dependency markers
+            if pkg_name.startswith(DEPENDENCY_MARKER_PREFIX):
+                continue
+            # Skip non-common-bus packages
+            if pkg_name not in common_bus_packages:
+                continue
+
+            # Find a component that has this package and extract its value
+            # Uses cached lookup to avoid re-loading the same files
+            for comp_name in component_names:
+                comp_packages = get_component_packages(
+                    comp_name, platform, tests_dir_str
+                )
+                if pkg_name in comp_packages:
+                    merged_packages[pkg_name] = comp_packages[pkg_name]
+                    break
+
+        if merged_packages:
+            merged_config_data["packages"] = merged_packages
 
     # Deduplicate items with same ID (keeps first occurrence)
     merged_config_data = deduplicate_by_id(merged_config_data)

script/test_build_components.py

@@ -32,6 +32,7 @@ from script.analyze_component_buses import (
     ISOLATED_COMPONENTS,
     NO_BUSES_SIGNATURE,
     analyze_all_components,
+    are_buses_compatible,
     create_grouping_signature,
     is_platform_component,
     uses_local_file_references,
@@ -357,6 +358,67 @@ def run_grouped_test(
         return False, cmd_str
 
 
+def merge_compatible_bus_groups(
+    grouped_components: dict[tuple[str, str], list[str]],
+) -> dict[tuple[str, str], list[str]]:
+    """Merge groups with compatible (non-conflicting) buses.
+
+    Two groups can be merged if they're on the same platform and their buses don't conflict.
+    For example: ["ble"] + ["uart"] = compatible, but ["uart"] + ["uart_9600"] = incompatible.
+
+    Args:
+        grouped_components: Dictionary mapping (platform, signature) to component list
+
+    Returns:
+        New dictionary with compatible groups merged together
+    """
+    merged_groups: dict[tuple[str, str], list[str]] = {}
+    processed_keys: set[tuple[str, str]] = set()
+
+    for (platform1, sig1), comps1 in sorted(grouped_components.items()):
+        if (platform1, sig1) in processed_keys:
+            continue
+
+        # Skip NO_BUSES_SIGNATURE for now - they'll be distributed later
+        if sig1 == NO_BUSES_SIGNATURE:
+            merged_groups[(platform1, sig1)] = comps1
+            processed_keys.add((platform1, sig1))
+            continue
+
+        # Start with this group's components
+        merged_comps = list(comps1)
+        merged_sig = sig1
+        processed_keys.add((platform1, sig1))
+
+        # Get buses for this group as tuple for caching
+        buses1 = tuple(sorted(sig1.split("+")))
+
+        # Try to merge with other groups on same platform
+        for (platform2, sig2), comps2 in sorted(grouped_components.items()):
+            if (platform2, sig2) in processed_keys:
+                continue
+            if platform2 != platform1:
+                continue  # Different platforms can't be merged
+            if sig2 == NO_BUSES_SIGNATURE:
+                continue  # Handle separately
+
+            # Check if buses are compatible
+            buses2 = tuple(sorted(sig2.split("+")))
+            if are_buses_compatible(buses1, buses2):
+                # Compatible! Merge this group
+                merged_comps.extend(comps2)
+                processed_keys.add((platform2, sig2))
+                # Update merged signature to include all unique buses
+                all_buses = set(buses1) | set(buses2)
+                merged_sig = "+".join(sorted(all_buses))
+                buses1 = tuple(sorted(all_buses))  # Update for next iteration
+
+        # Store merged group
+        merged_groups[(platform1, merged_sig)] = merged_comps
+
+    return merged_groups
+
+
 def run_grouped_component_tests(
     all_tests: dict[str, list[Path]],
     platform_filter: str | None,
@@ -462,6 +524,11 @@ def run_grouped_component_tests(
             if signature:
                 grouped_components[(platform, signature)].append(component)
 
+    # Merge groups with compatible buses (cross-bus grouping optimization)
+    # This allows mixing components with different buses (e.g., ble + uart)
+    # as long as they don't have conflicting configurations for the same bus type
+    grouped_components = merge_compatible_bus_groups(grouped_components)
+
     # Print detailed grouping plan
     print("\nGrouping Plan:")
     print("-" * 80)