diff --git a/src/redis_release/bht/behaviours.py b/src/redis_release/bht/behaviours.py index cd96a89..1d53c82 100644 --- a/src/redis_release/bht/behaviours.py +++ b/src/redis_release/bht/behaviours.py @@ -26,6 +26,7 @@ from redis_release.bht.state import reset_model_to_defaults from ..github_client_async import GitHubClientAsync +from ..logging_config import log_once from ..models import RedisVersion, ReleaseType, WorkflowConclusion, WorkflowStatus from .logging_wrapper import PyTreesLoggerWrapper from .state import Package, PackageMeta, ReleaseMeta, Workflow @@ -53,10 +54,7 @@ def log_exception_and_return_failure(self, e: Exception) -> Status: return Status.FAILURE def log_once(self, key: str, container: Dict[str, bool]) -> bool: - if key not in container: - container[key] = True - return True - return False + return log_once(key, container) class ReleaseAction(LoggingAction): diff --git a/src/redis_release/bht/behaviours_docker.py b/src/redis_release/bht/behaviours_docker.py index 5582df4..e3afc69 100644 --- a/src/redis_release/bht/behaviours_docker.py +++ b/src/redis_release/bht/behaviours_docker.py @@ -2,21 +2,17 @@ from py_trees.common import Status -from redis_release.bht.behaviours import LoggingAction, ReleaseAction -from redis_release.bht.state import PackageMeta, ReleaseMeta, Workflow -from redis_release.models import RedisVersion, ReleaseType +from ..models import RedisVersion, ReleaseType +from .behaviours import LoggingAction, ReleaseAction +from .state import DockerMeta, ReleaseMeta, Workflow class DockerWorkflowInputs(ReleaseAction): - """ - Docker uses only release_tag input which is set automatically in TriggerWorkflow - """ - def __init__( self, name: str, workflow: Workflow, - package_meta: PackageMeta, + package_meta: DockerMeta, release_meta: ReleaseMeta, log_prefix: str = "", ) -> None: @@ -26,6 +22,9 @@ def __init__( super().__init__(name=name, log_prefix=log_prefix) def update(self) -> Status: + if self.package_meta.module_versions is not None: + for module, version in self.package_meta.module_versions.items(): + self.workflow.inputs[f"{module.value}_version"] = version return Status.SUCCESS @@ -35,7 +34,7 @@ class DetectReleaseTypeDocker(LoggingAction): def __init__( self, name: str, - package_meta: PackageMeta, + package_meta: DockerMeta, release_meta: ReleaseMeta, log_prefix: str = "", ) -> None: @@ -52,7 +51,14 @@ def initialise(self) -> None: return if self.release_meta.tag == "unstable": return - self.release_version = RedisVersion.parse(self.release_meta.tag) + try: + self.release_version = RedisVersion.parse(self.release_meta.tag) + except ValueError as e: + if self.release_meta.tag != "": + self.logger.info( + f"Failed to parse release tag: {e}, assuming custom release with tag {self.release_meta.tag}" + ) + return def update(self) -> Status: result: Status = Status.FAILURE @@ -70,8 +76,9 @@ def update(self) -> Status: f"Detected release type for docker: {self.package_meta.release_type}" ) else: - self.feedback_message = "Failed to detect release type" - result = Status.FAILURE + self.package_meta.release_type = ReleaseType.INTERNAL + self.feedback_message = "Set release type to internal for custom build" + result = Status.SUCCESS if self.log_once( "release_type_detected", self.package_meta.ephemeral.log_once_flags @@ -92,7 +99,7 @@ class NeedToReleaseDocker(LoggingAction): def __init__( self, name: str, - package_meta: PackageMeta, + package_meta: DockerMeta, release_meta: ReleaseMeta, log_prefix: str = "", ) -> None: @@ -123,9 +130,6 @@ def update(self) -> Status: if self.release_meta.tag is None: self.feedback_message = "Release tag is not set" result = Status.FAILURE - if self.release_meta.tag == "unstable": - self.feedback_message = "Skip unstable release for docker" - result = Status.FAILURE if self.release_version is not None: if self.release_version.major < 8: @@ -138,6 +142,9 @@ def update(self) -> Status: f"Need to release docker version {str(self.release_version)}" ) result = Status.SUCCESS + else: + self.feedback_message = "Custom build, need to release" + result = Status.SUCCESS if self.log_once("need_to_release", self.package_meta.ephemeral.log_once_flags): color_open = "" if result == Status.SUCCESS else "[yellow]" diff --git a/src/redis_release/bht/behaviours_homebrew.py b/src/redis_release/bht/behaviours_homebrew.py index c24c426..b88549a 100644 --- a/src/redis_release/bht/behaviours_homebrew.py +++ b/src/redis_release/bht/behaviours_homebrew.py @@ -82,6 +82,12 @@ def update(self) -> Status: self.logger.info(self.feedback_message) return Status.SUCCESS + if self.release_version is None and self.release_meta.tag != "": + self.logger.info( + f"Release version is not set, skipping probably custom release {self.release_meta.tag}" + ) + return Status.SUCCESS + assert self.release_version is not None if self.package_meta.release_type is None: if self.release_version.is_internal: @@ -147,6 +153,12 @@ def initialise(self) -> None: self.package_meta.remote_version = "unstable" return + if self.release_meta.tag != "": + self.package_meta.ephemeral.is_version_acceptable = False + # we need to set remote version to not None as it is a sign of successful classify step + self.package_meta.remote_version = "custom" + return + self.feedback_message = "" # Validate homebrew_channel is set if self.package_meta.homebrew_channel is None: diff --git a/src/redis_release/bht/behaviours_snap.py b/src/redis_release/bht/behaviours_snap.py index 6b17e30..9ccb5c3 100644 --- a/src/redis_release/bht/behaviours_snap.py +++ b/src/redis_release/bht/behaviours_snap.py @@ -86,6 +86,11 @@ def update(self) -> Status: self.package_meta.release_type = ReleaseType.PUBLIC self.package_meta.snap_risk_level = SnapRiskLevel.EDGE else: + if self.release_version is None and self.release_meta.tag != "": + self.logger.info( + f"Release version is not set, skipping probably custom release: {self.release_meta.tag}" + ) + return Status.SUCCESS assert self.release_version is not None if self.package_meta.release_type is None: if self.release_version.is_internal: @@ -144,6 +149,18 @@ def initialise(self) -> None: if self.package_meta.ephemeral.is_version_acceptable is not None: return + # TODO: don't like this expresion, should be better way to handle custom releases skipping + if ( + self.release_meta.tag is not None + and self.release_meta.tag != "" + and self.release_meta.tag != "unstable" + and self.package_meta.snap_risk_level is None + ): + self.package_meta.ephemeral.is_version_acceptable = False + # we need to set remote version to not None as it is a sign of successful classify step + self.package_meta.remote_version = "custom" + return + self.feedback_message = "" # Validate snap_risk_level is set if self.package_meta.snap_risk_level is None: diff --git a/src/redis_release/bht/state.py b/src/redis_release/bht/state.py index 1784895..7dddf02 100644 --- a/src/redis_release/bht/state.py +++ b/src/redis_release/bht/state.py @@ -1,3 +1,60 @@ +"""State of release - central data model for the release process. + +Used by the behavior tree to keep track of the release progress, plays +blackboard role for behavior tree. + +Ephemeral and persistent fields +-------------------------------- + +The main purpose of ephemeral fields is to prevent retry loops and to allow +extensive status reporting. + +Each workflow step has a pair of fields indicating the step status: +One ephemeral field is set when the step is attempted. It may have four states: +- `None` (default): Step has not been attempted +- `common.Status.RUNNING`: Step is currently running +- `common.Status.FAILURE`: Step has been attempted and failed +- `common.Status.SUCCESS`: Step has been attempted and succeeded + +Ephemeral fields are reset on each run. Their values are persisted but only until +next run is started. +So they indicate either current (if run is in progress) or last run state. + +The other field indicates the step result, it may either have some value or be empty. +This field is persisted across runs. + +For example for trigger step we have `trigger_workflow` ephemeral +and `triggered_at` result fields. + +Optional message field may be used to provide additional information about the step. +For example wait_for_completion_message may contain information about timeout. + +Given combination of ephemeral and result fields we can determine step status. +Each step may be in one of the following states: + Not started + Failed + Succeeded or OK + Incorrect (this shouln't happen) + +The following decision table show how step status is determined for trigger step. +In general this is applicable to all steps. + +tigger_workflow -> | None (default) | Running | Failure | Success | +triggered_at: | | | | | + None | Not started | In progress | Failed | Incorrect | + Has value | OK | Incorrect | Incorrect | OK | + +The result field (triggered_at in this case) should not be set while step is +running, if step was not started or if it's failed. +And it should be set if trigger_workflow is successful. +It may be set if trigger_workflow is None, which is the case when release +process was restarted and all ephemeral fields are reset, but the particular +step was successful in previous run. + +Correct values are not eforced it's up to the implementation to correctly +set the fields. +""" + import json import logging from datetime import datetime @@ -8,9 +65,11 @@ from py_trees.common import Status from pydantic import BaseModel, Field -from redis_release.models import ( +from ..config import Config, PackageConfig +from ..models import ( HomebrewChannel, PackageType, + RedisModule, ReleaseType, SnapRiskLevel, WorkflowConclusion, @@ -18,62 +77,13 @@ WorkflowType, ) -from ..config import Config, PackageConfig - logger = logging.getLogger(__name__) -SUPPORTED_STATE_VERSION = 3 +SUPPORTED_STATE_VERSION = 4 class WorkflowEphemeral(BaseModel): - """Ephemeral workflow state. Reset on each run. - - The main purpose of ephemeral fields is to prevent retry loops and to allow extensive status reporting. - - Each workflow step has a pair of fields indicating the step status: - One ephemeral field is set when the step is attempted. It may have four states: - - `None` (default): Step has not been attempted - - `common.Status.RUNNING`: Step is currently running - - `common.Status.FAILURE`: Step has been attempted and failed - - `common.Status.SUCCESS`: Step has been attempted and succeeded - - Ephemeral fields are reset on each run. Their values are persisted but only until - next run is started. - So they indicate either current (if run is in progress) or last run state. - - The other field indicates the step result, it may either have some value or be empty. - - For example for trigger step we have `trigger_workflow` ephemeral - and `triggered_at` result fields. - - Optional message field may be used to provide additional information about the step. - For example wait_for_completion_message may contain information about timeout. - - Given combination of ephemeral and result fields we can determine step status. - Each step may be in one of the following states: - Not started - Failed - Succeeded or OK - Incorrect (this shouln't happen) - - The following decision table show how step status is determined for trigger step. - In general this is applicable to all steps. - - tigger_workflow -> | None (default) | Running | Failure | Success | - triggered_at: | | | | | - None | Not started | In progress | Failed | Incorrect | - Has value | OK | Incorrect | Incorrect | OK | - - The result field (triggered_at in this case) should not be set while step is - running, if step was not started or if it's failed. - And it should be set if trigger_workflow is successful. - It may be set if trigger_workflow is None, which is the case when release - process was restarted and all ephemeral fields are reset, but the particular - step was successful in previous run. - - Correct values are not eforced it's up to the implementation to correctly - set the fields. - """ + """Ephemeral workflow state. Reset on each run.""" identify_workflow: Optional[common.Status] = None trigger_workflow: Optional[common.Status] = None @@ -138,6 +148,10 @@ class SnapMetaEphemeral(PackageMetaEphemeral): pass +class DockerMetaEphemeral(PackageMetaEphemeral): + pass + + class PackageMeta(BaseModel): """Metadata for a package (base/generic type).""" @@ -176,6 +190,14 @@ class SnapMeta(PackageMeta): ephemeral: SnapMetaEphemeral = Field(default_factory=SnapMetaEphemeral) # type: ignore[assignment] +class DockerMeta(PackageMeta): + """Metadata for Docker package.""" + + serialization_hint: Literal["docker"] = "docker" # type: ignore[assignment] + module_versions: Optional[Dict[RedisModule, str]] = None + ephemeral: DockerMetaEphemeral = Field(default_factory=DockerMetaEphemeral) # type: ignore[assignment] + + class Package(BaseModel): """State for a package in the release. @@ -186,7 +208,7 @@ class Package(BaseModel): - serialization_hint="snap" -> SnapMeta """ - meta: Union[HomebrewMeta, SnapMeta, PackageMeta] = Field( + meta: Union[HomebrewMeta, SnapMeta, PackageMeta, DockerMeta] = Field( default_factory=PackageMeta, discriminator="serialization_hint" ) build: Workflow = Field(default_factory=Workflow) @@ -227,7 +249,7 @@ def _create_package_meta_from_config( package_config: Package configuration Returns: - PackageMeta subclass instance (HomebrewMeta, SnapMeta, or PackageMeta) + PackageMeta subclass instance (HomebrewMeta, SnapMeta, DockerMeta or PackageMeta) Raises: ValueError: If package_type is None @@ -246,6 +268,13 @@ def _create_package_meta_from_config( package_type=package_config.package_type, publish_internal_release=package_config.publish_internal_release, ) + elif package_config.package_type == PackageType.DOCKER: + return DockerMeta( + repo=package_config.repo, + ref=package_config.ref, + package_type=package_config.package_type, + publish_internal_release=package_config.publish_internal_release, + ) elif package_config.package_type is not None: return PackageMeta( repo=package_config.repo, diff --git a/src/redis_release/bht/tree_factory_docker.py b/src/redis_release/bht/tree_factory_docker.py index 0864d09..2918bc5 100644 --- a/src/redis_release/bht/tree_factory_docker.py +++ b/src/redis_release/bht/tree_factory_docker.py @@ -1,12 +1,14 @@ +from typing import cast + from py_trees.behaviour import Behaviour -from redis_release.bht.behaviours_docker import ( +from .behaviours_docker import ( DetectReleaseTypeDocker, DockerWorkflowInputs, NeedToReleaseDocker, ) -from redis_release.bht.state import PackageMeta, ReleaseMeta, Workflow -from redis_release.bht.tree_factory_generic import GenericPackageFactory +from .state import DockerMeta, PackageMeta, ReleaseMeta, Workflow +from .tree_factory_generic import GenericPackageFactory class DockerFactory(GenericPackageFactory): @@ -21,7 +23,11 @@ def create_build_workflow_inputs( log_prefix: str, ) -> Behaviour: return DockerWorkflowInputs( - name, workflow, package_meta, release_meta, log_prefix=log_prefix + name, + workflow, + cast(DockerMeta, package_meta), + release_meta, + log_prefix=log_prefix, ) def create_publish_workflow_inputs( @@ -33,7 +39,11 @@ def create_publish_workflow_inputs( log_prefix: str, ) -> Behaviour: return DockerWorkflowInputs( - name, workflow, package_meta, release_meta, log_prefix=log_prefix + name, + workflow, + cast(DockerMeta, package_meta), + release_meta, + log_prefix=log_prefix, ) def create_need_to_release_behaviour( @@ -44,7 +54,7 @@ def create_need_to_release_behaviour( log_prefix: str, ) -> Behaviour: return NeedToReleaseDocker( - name, package_meta, release_meta, log_prefix=log_prefix + name, cast(DockerMeta, package_meta), release_meta, log_prefix=log_prefix ) def create_detect_release_type_behaviour( @@ -55,5 +65,5 @@ def create_detect_release_type_behaviour( log_prefix: str, ) -> Behaviour: return DetectReleaseTypeDocker( - name, package_meta, release_meta, log_prefix=log_prefix + name, cast(DockerMeta, package_meta), release_meta, log_prefix=log_prefix ) diff --git a/src/redis_release/cli.py b/src/redis_release/cli.py index 377e753..58d8020 100644 --- a/src/redis_release/cli.py +++ b/src/redis_release/cli.py @@ -3,19 +3,21 @@ import asyncio import logging import os -from typing import Dict, List, Optional +from typing import List, Optional import typer from openai import OpenAI from py_trees.display import render_dot_tree, unicode_tree +from redis_release.cli_util import parse_force_release_type, parse_module_versions + from .bht.conversation_state import InboxMessage from .bht.conversation_tree import initialize_conversation_tree from .bht.tree import TreeInspector, async_tick_tock, initialize_tree_and_state from .config import load_config from .conversation_models import ConversationArgs, InboxMessage from .logging_config import setup_logging -from .models import ReleaseArgs, ReleaseType, SlackArgs +from .models import RedisModule, ReleaseArgs, SlackArgs from .state_display import print_state_table from .state_manager import InMemoryStateStorage, S3StateStorage, StateManager from .state_slack import init_slack_printer @@ -29,47 +31,6 @@ logger = logging.getLogger(__name__) -def parse_force_release_type( - force_release_type_list: Optional[List[str]], -) -> Dict[str, ReleaseType]: - """Parse force_release_type arguments from 'package_name:release_type' format. - - Args: - force_release_type_list: List of strings in format 'package_name:release_type' - - Returns: - Dictionary mapping package names to ReleaseType - - Raises: - typer.BadParameter: If format is invalid or release type is unknown - """ - if not force_release_type_list: - return {} - - result = {} - for item in force_release_type_list: - if ":" not in item: - raise typer.BadParameter( - f"Invalid format '{item}'. Expected 'package_name:release_type' (e.g., 'docker:internal')" - ) - - package_name, release_type_str = item.split(":", 1) - package_name = package_name.strip() - release_type_str = release_type_str.strip().lower() - - try: - release_type = ReleaseType(release_type_str) - except ValueError: - valid_types = ", ".join([rt.value for rt in ReleaseType]) - raise typer.BadParameter( - f"Invalid release type '{release_type_str}'. Valid types: {valid_types}" - ) - - result[package_name] = release_type - - return result - - @app.command() def release_print( release_tag: str = typer.Argument(..., help="Release tag (e.g., 8.4-m01-int1)"), @@ -158,6 +119,11 @@ def release( "--only-packages", help="Only process specific packages (can be specified multiple times)", ), + module_versions: Optional[List[str]] = typer.Option( + None, + "--module-version", + help="Specific module version to use (e.g., 'redisjson:2.4.0'). Can be specified multiple times.", + ), tree_cutoff: int = typer.Option( 5000, "--tree-cutoff", "-m", help="Max number of ticks to run the tree for" ), @@ -194,6 +160,7 @@ def release( only_packages=only_packages or [], force_release_type=parse_force_release_type(force_release_type), override_state_name=override_state_name, + module_versions=parse_module_versions(module_versions), slack_args=SlackArgs( bot_token=slack_token, channel_id=slack_channel_id, diff --git a/src/redis_release/cli_util.py b/src/redis_release/cli_util.py new file mode 100644 index 0000000..efc95e2 --- /dev/null +++ b/src/redis_release/cli_util.py @@ -0,0 +1,92 @@ +from typing import Dict, List, Optional + +from typer import BadParameter + +from redis_release.models import RedisModule, ReleaseType + + +def parse_force_release_type( + force_release_type_list: Optional[List[str]], +) -> Dict[str, ReleaseType]: + """Parse force_release_type arguments from 'package_name:release_type' format. + + Args: + force_release_type_list: List of strings in format 'package_name:release_type' + + Returns: + Dictionary mapping package names to ReleaseType + + Raises: + BadParameter: If format is invalid or release type is unknown + """ + if not force_release_type_list: + return {} + + result = {} + for item in force_release_type_list: + if ":" not in item: + raise BadParameter( + f"Invalid format '{item}'. Expected 'package_name:release_type' (e.g., 'docker:internal')" + ) + + package_name, release_type_str = item.split(":", 1) + package_name = package_name.strip() + release_type_str = release_type_str.strip().lower() + + try: + release_type = ReleaseType(release_type_str) + except ValueError: + valid_types = ", ".join([rt.value for rt in ReleaseType]) + raise BadParameter( + f"Invalid release type '{release_type_str}'. Valid types: {valid_types}" + ) + + result[package_name] = release_type + + return result + + +def parse_module_versions( + module_versions_list: Optional[List[str]], +) -> Dict[RedisModule, str]: + """Parse module versions from 'module_name:version' format. + + Args: + module_versions_list: List of strings in format 'module_name:version' + + Returns: + Dictionary mapping RedisModule to version strings + + Raises: + BadParameter: If format is invalid or module name is unknown + """ + if not module_versions_list: + return {} + + result = {} + for item in module_versions_list: + if ":" not in item: + raise BadParameter( + f"Invalid format '{item}'. Expected 'module_name:version' (e.g., 'redisjson:2.6.0')" + ) + + module_name, version = item.split(":", 1) + module_name = module_name.strip().lower() + version = version.strip() + + # Find matching RedisModule enum value + module_enum = None + for module in RedisModule: + if module.value.lower() == module_name: + module_enum = module + break + + if module_enum is None: + valid_modules = ", ".join([m.value for m in RedisModule]) + raise BadParameter( + f"Invalid module name '{module_name}'. Valid modules: {valid_modules}" + ) + + result[module_enum] = version + + return result diff --git a/src/redis_release/logging_config.py b/src/redis_release/logging_config.py index 1f439d3..1711c85 100644 --- a/src/redis_release/logging_config.py +++ b/src/redis_release/logging_config.py @@ -2,7 +2,7 @@ import logging import os -from typing import Optional +from typing import Dict, Optional from rich.logging import RichHandler @@ -104,3 +104,10 @@ def setup_logging( logging.getLogger("botocore").setLevel(third_party_level) logging.getLogger("boto3").setLevel(third_party_level) logging.getLogger("urllib3").setLevel(third_party_level) + + +def log_once(key: str, container: Dict[str, bool]) -> bool: + if key not in container: + container[key] = True + return True + return False diff --git a/src/redis_release/models.py b/src/redis_release/models.py index dc9b089..758229c 100644 --- a/src/redis_release/models.py +++ b/src/redis_release/models.py @@ -71,6 +71,15 @@ class WorkflowConclusion(str, Enum): FAILURE = "failure" +class RedisModule(str, Enum): + """Redis module enumeration.""" + + JSON = "redisjson" + SEARCH = "redisearch" + TIMESERIES = "reduistimeseries" + BLOOM = "redisbloom" + + class WorkflowRun(BaseModel): """Represents a GitHub workflow run.""" @@ -209,6 +218,7 @@ class ReleaseArgs(BaseModel): only_packages: List[str] = Field(default_factory=list) force_release_type: Dict[str, ReleaseType] = Field(default_factory=dict) override_state_name: Optional[str] = None + module_versions: Dict[RedisModule, str] = Field(default_factory=dict) slack_args: Optional["SlackArgs"] = None diff --git a/src/redis_release/state_manager.py b/src/redis_release/state_manager.py index c24a381..8b92321 100644 --- a/src/redis_release/state_manager.py +++ b/src/redis_release/state_manager.py @@ -14,7 +14,8 @@ from redis_release.bht.state import ReleaseState, logger from redis_release.config import Config -from .bht.state import ReleaseState +from .bht.state import DockerMeta, PackageType, ReleaseState +from .logging_config import log_once from .models import ReleaseArgs logger = logging.getLogger(__name__) @@ -221,10 +222,10 @@ def state(self) -> ReleaseState: def default_state(self) -> ReleaseState: """Create default state from config.""" state = ReleaseState.from_config(self.config) - self.apply_args(state) + self.apply_args(state, quiet=True) return state - def apply_args(self, state: ReleaseState) -> None: + def apply_args(self, state: ReleaseState, quiet: bool = False) -> None: """Apply arguments to state.""" state.meta.tag = self.tag @@ -240,7 +241,8 @@ def apply_args(self, state: ReleaseState) -> None: state.packages[package_name].meta.ephemeral.force_rebuild = True if self.args.force_release_type: - logger.info(f"Force release type: {self.args.force_release_type}") + if not quiet: + logger.info(f"Force release type: {self.args.force_release_type}") # Handle "all" keyword to apply to all packages if "all" in self.args.force_release_type: release_type = self.args.force_release_type["all"] @@ -266,6 +268,16 @@ def apply_args(self, state: ReleaseState) -> None: logger.warning( f"Package '{package_name}' not found in state, skipping release type override" ) + if self.args.module_versions: + for package in state.packages.values(): + if package.meta.package_type == PackageType.DOCKER: + assert isinstance(package.meta, DockerMeta) + if not quiet: + for module, version in self.args.module_versions.items(): + logger.info( + f"Set module version for [yellow]{module}: {version}[/]" + ) + package.meta.module_versions = self.args.module_versions def load(self) -> Optional[ReleaseState]: """Load state from storage backend."""