From 4f8f4591ca35f23481ffcab76789cf62318a00c4 Mon Sep 17 00:00:00 2001 From: lomnido <135008914+lomnido@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:53:48 +0200 Subject: [PATCH] introducing 'dump-manifest' command it can: * create Manifest file out of given Repos (RAW mode) * create Manifest file out of current Workspace * also update existing Manifest file (this keeps the Groups there) * update Deep Manifest (auto-detection) also in regard to other changes: * '(missing remote)' to display for Deep Manifest and Future Manifest * allowing to ignore missing Group's items * 'status' can speeded up by ignoring GIT remote data --- tsrc/cli/__init__.py | 87 +- tsrc/cli/dump_manifest.py | 339 +++++ tsrc/cli/main.py | 12 +- tsrc/cli/manifest.py | 30 +- tsrc/cli/status.py | 55 +- tsrc/cli/sync.py | 24 +- tsrc/dump_manifest.py | 1154 +++++++++++++++ tsrc/dump_manifest_args.py | 125 ++ tsrc/dump_manifest_args_data.py | 123 ++ tsrc/dump_manifest_args_final_output.py | 65 + tsrc/dump_manifest_args_source_mode.py | 57 + tsrc/dump_manifest_args_update_source.py | 144 ++ tsrc/dump_manifest_helper.py | 100 ++ tsrc/dump_manifest_raw_grabber.py | 172 +++ tsrc/errors.py | 4 +- tsrc/file_system.py | 10 + tsrc/git.py | 23 - tsrc/git_remote.py | 52 + tsrc/groups.py | 17 +- tsrc/groups_and_constraints_data.py | 36 + tsrc/manifest_common_data.py | 15 +- tsrc/repo.py | 11 +- tsrc/repo_grabber.py | 71 + tsrc/status_endpoint.py | 84 +- tsrc/status_header.py | 7 +- tsrc/test/cli/test_dump_manifest.py | 1455 +++++++++++++++++++ tsrc/test/cli/test_dump_manifest__args.py | 667 +++++++++ tsrc/test/cli/test_dump_manifest__groups.py | 617 ++++++++ tsrc/test/cli/test_groups_extra.py | 3 +- tsrc/test/cli/test_sync_extended.py | 2 +- tsrc/test/cli/test_sync_to_ref.py | 22 +- tsrc/test/conftest.py | 9 + tsrc/test/helpers/git_server.py | 15 +- tsrc/test/helpers/manifest_file.py | 4 +- tsrc/test/helpers/message_recorder_ext.py | 76 + tsrc/workspace.py | 26 +- tsrc/workspace_repos_summary.py | 40 +- 37 files changed, 5644 insertions(+), 109 deletions(-) create mode 100644 tsrc/cli/dump_manifest.py create mode 100644 tsrc/dump_manifest.py create mode 100644 tsrc/dump_manifest_args.py create mode 100644 tsrc/dump_manifest_args_data.py create mode 100644 tsrc/dump_manifest_args_final_output.py create mode 100644 tsrc/dump_manifest_args_source_mode.py create mode 100644 tsrc/dump_manifest_args_update_source.py create mode 100644 tsrc/dump_manifest_helper.py create mode 100644 tsrc/dump_manifest_raw_grabber.py create mode 100644 tsrc/groups_and_constraints_data.py create mode 100644 tsrc/repo_grabber.py create mode 100644 tsrc/test/cli/test_dump_manifest.py create mode 100644 tsrc/test/cli/test_dump_manifest__args.py create mode 100644 tsrc/test/cli/test_dump_manifest__groups.py create mode 100644 tsrc/test/helpers/message_recorder_ext.py diff --git a/tsrc/cli/__init__.py b/tsrc/cli/__init__.py index 4b66c789..532b481d 100644 --- a/tsrc/cli/__init__.py +++ b/tsrc/cli/__init__.py @@ -11,7 +11,9 @@ import cli_ui as ui from tsrc.errors import Error +from tsrc.groups_and_constraints_data import GroupsAndConstraints from tsrc.manifest import Manifest +from tsrc.manifest_common_data import ManifestsTypeOfData from tsrc.repo import Repo from tsrc.workspace import Workspace from tsrc.workspace_config import WorkspaceConfig @@ -105,6 +107,7 @@ def find_workspace_path() -> Path: def get_workspace_with_repos( namespace: argparse.Namespace, ignore_if_group_not_found: bool = False, + ignore_group_item: bool = False, ) -> Workspace: workspace = get_workspace(namespace, silent=ignore_if_group_not_found) workspace.repos = resolve_repos( @@ -114,6 +117,7 @@ def get_workspace_with_repos( include_regex=namespace.include_regex, exclude_regex=namespace.exclude_regex, ignore_if_group_not_found=ignore_if_group_not_found, + ignore_group_item=ignore_group_item, ) return workspace @@ -129,6 +133,7 @@ def simulate_get_workspace_with_repos( include_regex=namespace.include_regex, exclude_regex=namespace.exclude_regex, ignore_if_group_not_found=True, + ignore_group_item=True, ) @@ -141,13 +146,17 @@ def simulate_resolve_repos( include_regex: str = "", exclude_regex: str = "", ignore_if_group_not_found: bool = False, + ignore_group_item: bool = False, ) -> List[str]: """ just to obatin 'groups_seen' as if we hit the exception, we may miss some groups """ # Handle --all-cloned and --groups - manifest = workspace.get_manifest() + if ignore_group_item is True: + manifest = workspace.get_manifest_safe_mode(ManifestsTypeOfData.LOCAL) + else: + manifest = workspace.get_manifest() if groups: manifest.get_repos(groups=groups, ignore_if_group_not_found=True) @@ -165,6 +174,7 @@ def resolve_repos( include_regex: str = "", exclude_regex: str = "", ignore_if_group_not_found: bool = False, + ignore_group_item: bool = False, ) -> List[Repo]: """ Given a workspace with its config and its local manifest, @@ -172,7 +182,10 @@ def resolve_repos( return the list of repositories to operate on. """ # Handle --all-cloned and --groups - manifest = workspace.get_manifest() + if ignore_group_item is True: + manifest = workspace.get_manifest_safe_mode(ManifestsTypeOfData.LOCAL) + else: + manifest = workspace.get_manifest() repos = [] if groups: @@ -210,6 +223,76 @@ def resolve_repos( return repos +def resolve_repos_without_workspace( + manifest: Manifest, + gac: GroupsAndConstraints, +) -> List[Repo]: + """ + Use just Manifest to get Repos in regard of Groups, + include_regex, exclude_regex. Also respect 'singular_remote' + If no Groups are provided, consider all Repos there are. + Return Repos to operate on. + """ + repos = [] + + if gac.groups: + # due to we are working with Manifest, there is + # no reason to enforce group to exist + repos = manifest.get_repos(groups=gac.groups, ignore_if_group_not_found=True) + else: + repos = manifest.get_repos(all_=True) + + if gac.singular_remote: + filtered_repos = [] + for repo in repos: + remotes = [ + remote + for remote in repo.remotes + if gac.singular_remote.lower() == remote.name.lower() + ] + if remotes: + filtered_repos.append(repo) + repos = filtered_repos + + if gac.include_regex: + repos = [repo for repo in repos if re.search(gac.include_regex, repo.dest)] + + if gac.exclude_regex: + repos = [repo for repo in repos if not re.search(gac.exclude_regex, repo.dest)] + + return repos + + +def resolve_repos_apply_constraints( + repos: List[Repo], + gac: GroupsAndConstraints, +) -> List[Repo]: + """ + Use just constraints on Repos in GroupAndConstraints class + to filter Repos. Consider: + include_regex, exclude_regex. Also respect 'singular_remote' + """ + if gac.singular_remote: + filtered_repos = [] + for repo in repos: + remotes = [ + remote + for remote in repo.remotes + if gac.singular_remote.lower() == remote.name.lower() + ] + if remotes: + filtered_repos.append(repo) + repos = filtered_repos + + if gac.include_regex: + repos = [repo for repo in repos if re.search(gac.include_regex, repo.dest)] + + if gac.exclude_regex: + repos = [repo for repo in repos if not re.search(gac.exclude_regex, repo.dest)] + + return repos + + def repos_from_config( manifest: Manifest, workspace_config: WorkspaceConfig, diff --git a/tsrc/cli/dump_manifest.py b/tsrc/cli/dump_manifest.py new file mode 100644 index 00000000..d7b37190 --- /dev/null +++ b/tsrc/cli/dump_manifest.py @@ -0,0 +1,339 @@ +""" Entry point for `tsrc dump-manifest`. + +This is actually one of the few commands that does not need to have Workspace. +In fact, we can start the project with this command by using RAW dump +to create current Manifest file, which we can put to new GIT Repository, +push it to remote and then calls 'tsrc init'. + +RAW dump (mode of operation): + RAW dump means we are creating/updating Manifest WITHOUT Workspace. + So we do not have '.tscr/config' or any other '.tsrc' data + (not even Groups if they are not present in Manifest when we updating it) + +Siplest way to start a new 'tsrc' project by creating Manifest is to prepare +every repository into some dedicated directory and from there call: + +'tsrc dump-manifest --raw .' + +Which creates Manifest file there. Which you can place into some other +directory, push to some remote and call: + +'tsrc init ' + +Which will create 'tsrc' Workspace right there. +""" + +import argparse +import io +from pathlib import Path +from typing import Dict, List, Tuple, Union + +import cli_ui as ui +from ruamel.yaml import YAML + +from tsrc.cli import ( + add_num_jobs_arg, + add_repos_selection_args, + add_workspace_arg, + get_num_jobs, +) +from tsrc.dump_manifest import ManifestDumper +from tsrc.dump_manifest_args import DumpManifestArgs +from tsrc.dump_manifest_args_data import ( + FinalOutputModeFlag, + SourceModeEnum, + UpdateSourceEnum, +) +from tsrc.dump_manifest_helper import MRISHelpers +from tsrc.dump_manifest_raw_grabber import ManifestRawGrabber +from tsrc.errors import MissingRepoError +from tsrc.executor import process_items +from tsrc.file_system import make_relative +from tsrc.repo import Repo +from tsrc.status_endpoint import CollectedStatuses, StatusCollector +from tsrc.utils import erase_last_line + + +def configure_parser(subparser: argparse._SubParsersAction) -> None: + parser = subparser.add_parser( + "dump-manifest", + description="Dump Manifest by obtaining data from one of these 2 MODES: from RAW SOURCE or from Workspace. Optionaly use obtained data to UPDATE exising YAML file (can also update Deep Manifest). And at the end, write output to DESTINATION considering PROPERTIES. By default Workspace MODE is used as data source and 'manifest.yml' in Workspace root path is used as DESTINATION. No UPDATE is done by default. 'manifest.yml' filename is default everytime directory is provided, but the file is required.", # noqa: E501 + ) + parser.add_argument( + "-r", + "--raw", + type=Path, + help="WARNING: for this, execution Path DOES matter. It switch MODE to RAW dump, settig SOURCE to provided Path (relative to WORKSPACE_PATH if set, or to execution Path otherwise) to search for any GIT repositories recursively to be used as data source. by default DESTINATION is 'manifest.yml' in COMMON PATH. COMMON PATH is calculated during execution time on given directory structure of where Repos are located as the deepest common root of all Repos while keeping each Repo directory its own", # noqa: E501 + dest="raw_dump_path", + ) + parser.add_argument( + "-u", + "--update", + action="store_true", + help="Regardles of operation mode, it will always look for Deep Manifest and set it as UPDATE source and DESTINATION default", # noqa: E501 + dest="do_update", + ) + parser.add_argument( + "-U", + "--update-on", + help="Set UPDATE operation mode, by setting the UPDATE source and DESTINATION default to provided UPDATE_AT path to YAML file. Such path must exists", # noqa: E501 + type=Path, + dest="update_on", + ) + parser.add_argument( + "--no-repo-delete", + action="store_true", + help="Disallow to delete any Repo record from existing Manifest. This have only meaning when on UPDATE operation mode", # noqa: E501 + dest="no_repo_delete", + ) + parser.add_argument( + "-p", + "--preview", + action="store_true", + help="Set DESTINATION to stdout ignoring any previous DESTINATION defaults. This option has the higher priority out of all modifying DESTINATION. No filesystem write operation will be made", # noqa: E501 + dest="just_preview", + ) + parser.add_argument( + "-s", + "--save-to", + help="Set DESTINATION to Path or Filename, ignoring any previous DESTINATION defaults. if Path is directory, it will be extended by default Manifest's filename", # noqa: E501 + type=Path, + dest="save_to", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Set PROPERTY to allow dangerous operations, like overwrite an already existing file. Use with care", # noqa: E501 + dest="use_force", + ) + add_workspace_arg(parser) + add_repos_selection_args(parser) + add_num_jobs_arg(parser) + parser.set_defaults(run=run) + + +def run(args: argparse.Namespace) -> None: + + try: + # checking args and preparing its data + a = DumpManifestArgs(args) + + # if no Exception was hit, we can continue + # to actual processing the command + ddm = DumpManifestsCMDLogic(a, num_jobs=get_num_jobs(args)) + + ddm.get_data() + + if ddm.yy: + ddm.output_data() + except Exception as e: + ui.error(e) + finally: + if "a" in locals(): + a.dmod.clean() + + +class DumpManifestsCMDLogic: + """ + Check and prepare arguments, perform operations on filesystem (if needed), + everything but not actual YAML data handling and/or processing. + For that the 'DumpManifest' and 'ManifestRawGrabber' classes are dedicated. + """ + + def __init__(self, a: DumpManifestArgs, num_jobs: int) -> None: + # default presets for output YAML + self.yaml = YAML() # for final output + self.yaml.indent(mapping=2, sequence=4, offset=2) + self.yaml.allow_duplicate_keys = True + self.yaml.default_flow_style = False # this makes a huge difference + + # output data + self.yy: Union[Dict, List, None] = None + self.load_path: Union[Path, None] = None # may be useless + self.save_path: Union[Path, None] = None # may be useless + self.is_updated: Union[bool, None] = None + + # everything in regard of args + self.a = a + + # translate MRISHelpers's data into YAML-useful data + # it can also update on existing data + self.m_du = ManifestDumper() + self.num_jobs = num_jobs + + """ + ===== 'get_data' section ===== + """ + + def get_data(self) -> None: + + # prepare 'mris_h' dataclass + try: + if self.a.dmod.source_mode == SourceModeEnum.RAW_DUMP: + repos = self._get_data_get_repos() + if not repos: + raise Exception("cannot obtain data: no Repos were found") + self.mris_h = MRISHelpers(repos=repos) + elif self.a.dmod.source_mode == SourceModeEnum.WORKSPACE_DUMP: + statuses, w_repos = self._get_data_get_statuses() + for status in statuses: + if not ( + isinstance(status, MissingRepoError) + or isinstance(status, Exception) # noqa: W503 + ): + break + else: + raise Exception("cannot obtain data: no useful Repos were found") + self.mris_h = MRISHelpers(statuses=statuses, w_repos=w_repos) + else: + raise Exception("cannot detect dump operation mode") + except Exception as e: + raise (e) + + # only up to this Fn the 'statuses' and 'repos' are relevant + # we will work with 'mris_h' right after this comment + + self._get_yaml_data() + + def _get_data_get_repos(self) -> List[Repo]: + + # grab Repos + try: + if self.a.dmod.source_path: + mgr = ManifestRawGrabber(self.a, self.a.dmod.source_path) + repos, self.a = mgr.grab(self.num_jobs) + except Exception as e: + raise (e) + + return repos + + def _get_data_get_statuses(self) -> Tuple[CollectedStatuses, List[Repo]]: + if self.a.dmod.workspace: + status_collector = StatusCollector(self.a.dmod.workspace) + w_repos = self.a.dmod.workspace.repos + if not w_repos: + raise Exception("Workspace is empty, therefore no valid data") + ui.info_1(f"Collecting statuses of {len(w_repos)} repo(s)") + process_items(w_repos, status_collector, num_jobs=self.num_jobs) + erase_last_line() + return status_collector.statuses, w_repos + return {}, [] + + def _get_yaml_data(self) -> None: + + # decided: go Update + if self.a.dmod.update_source != UpdateSourceEnum.NONE: + + # we need to get ready to Load the YAML file + y = None # loaded data + + if self.a.dmod.final_output_path_list.update_on_path: + with self.a.dmod.final_output_path_list.update_on_path.open( + "r" + ) as opened_file: + y = self.yaml.load(opened_file) + else: + raise Exception("Cannot obtain Manifest Repo from Workspace to update") + + if y: + self.yy, self.is_updated = self.m_du.on_update( + y, + self.mris_h.mris, + self.a.mdo, + self.a.gac, + ) + + if not self.yy: + raise Exception( + f"Not able to load YAML data from file: '{self.load_path}'" + ) + + else: # decided: create Manifest YAML data (not loading YAML data) + self.yy = self.m_du.do_create(self.mris_h.mris) + if self.yy: + self.is_updated = True + + """ + ===== 'output_data' section ===== + """ + + def output_data(self) -> None: + + if FinalOutputModeFlag.PREVIEW in self.a.dmod.final_output_mode: + + self._output_preview() + + if not self.is_updated or self.is_updated is False: + ui.warning("There was no change detected") + + else: + # do not write anything if there is no actual change + if not self.is_updated or self.is_updated is False: + ui.warning("Nothing has been changed, skipping") + + else: + + # save data + + self._output_data_ready_save_path() + + def _output_preview(self) -> None: + + # use 'ui.info' instead of real STDOUT + # this helps tests to catch the output + buff = io.BytesIO() + self.yaml.dump(self.yy, buff) + o_buff = buff.getvalue().decode("utf-8").splitlines(False) + for x in o_buff: + ui.info(x) + + def _output_data_ready_save_path(self) -> None: + + message: List[ui.Token] = [] + save_path: Union[Path, None] = None + + first_part: str = "" + second_part: str = "" + + if FinalOutputModeFlag.NEW in self.a.dmod.final_output_mode: + save_path = self.a.dmod.get_path_for_new() + first_part = f"Creating NEW file '{save_path}'" + elif FinalOutputModeFlag.OVERWRITE in self.a.dmod.final_output_mode: + save_path = self.a.dmod.get_path_for_new() + first_part = f"OVERWRITING file '{save_path}'" + + # UPDATE mode can be combined with NEW|OVERWRITE + if FinalOutputModeFlag.UPDATE in self.a.dmod.final_output_mode: + # only for UPDATE + update_path = self.a.dmod.update_source_path + if update_path: + update_path = make_relative(update_path) + save_path = self.a.dmod.get_path_for_update() + if self.a.dmod.update_source == UpdateSourceEnum.DEEP_MANIFEST: + second_part = f"UPDATING Deep Manifest on '{update_path}'" + elif self.a.dmod.update_source == UpdateSourceEnum.FILE: + second_part = f"UPDATING '{update_path}'" + + if first_part and second_part: + first_part += " by " + + # take care of printing correct message + message = [first_part + second_part] + ui.info_2(*message) + + # take care of obtained save_path + self._output_data_perform_dump(save_path) + + def _output_data_perform_dump(self, save_path: Union[Path, None]) -> None: + + # we are taking whole data, no constraints are considered. + if self.m_du.some_remote_is_missing(self.yy) is True: + ui.warning("This Manifest is not useful due to some missing remotes") + + if save_path: + with save_path.open("w") as stream: + self.yaml.dump(self.yy, stream) + ui.info_1("Dump complete") # only in this case: report success + else: + raise Exception("cannot find the desired path where to write Manifest") diff --git a/tsrc/cli/main.py b/tsrc/cli/main.py index 73e31d1d..9dccd2ca 100644 --- a/tsrc/cli/main.py +++ b/tsrc/cli/main.py @@ -10,7 +10,16 @@ import colored_traceback from tsrc import __version__ -from tsrc.cli import apply_manifest, foreach, init, log, manifest, status, sync +from tsrc.cli import ( + apply_manifest, + dump_manifest, + foreach, + init, + log, + manifest, + status, + sync, +) from tsrc.errors import Error ArgsList = Optional[Sequence[str]] @@ -85,6 +94,7 @@ def main_impl(args: ArgsList = None) -> None: for module in ( apply_manifest, + dump_manifest, foreach, init, log, diff --git a/tsrc/cli/manifest.py b/tsrc/cli/manifest.py index d62c71a5..484620aa 100644 --- a/tsrc/cli/manifest.py +++ b/tsrc/cli/manifest.py @@ -21,6 +21,7 @@ # from tsrc.status_footer import StatusFooter from tsrc.status_header import StatusHeader, StatusHeaderDisplayMode +from tsrc.utils import erase_last_line from tsrc.workspace_repos_summary import WorkspaceReposSummary @@ -67,6 +68,18 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: help="do not check for leftover's GIT descriptions", dest="strict_on_git_desc", ) + parser.add_argument( + "--ignore-missing-groups", + action="store_true", + dest="ignore_if_group_not_found", + help="ignore configured group(s) if it is not found in groups defined in manifest. This may be particulary useful when switching Manifest version back when some Groups defined later, was not there yet. In which case we can avoid unecessary Error caused by missing group", # noqa: E501 + ) + parser.add_argument( + "--ignore-missing-group-items", + action="store_true", + dest="ignore_group_item", + help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use", # noqa: E501 + ) parser.set_defaults(run=run) @@ -76,13 +89,19 @@ def run(args: argparse.Namespace) -> None: gtf.found_these(groups_seen) try: - workspace = get_workspace_with_repos(args) + workspace = get_workspace_with_repos( + args, ignore_group_item=args.ignore_group_item + ) except GroupNotFound: # try to obtain workspace ignoring group error # if group is found in Deep Manifest or Future Manifest, # do not report GroupNotFound. # if not, than raise exception at the very end - workspace = get_workspace_with_repos(args, ignore_if_group_not_found=True) + workspace = get_workspace_with_repos( + args, + ignore_if_group_not_found=True, + ignore_group_item=args.ignore_group_item, + ) dm = None if args.use_deep_manifest is True: @@ -114,7 +133,9 @@ def run(args: argparse.Namespace) -> None: cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH] ) status_header.display() - status_collector = StatusCollector(workspace) + status_collector = StatusCollector( + workspace, ignore_group_item=args.ignore_group_item + ) repos = deepcopy(workspace.repos) @@ -129,6 +150,7 @@ def run(args: argparse.Namespace) -> None: # num_jobs=1 as we will have only (max) 1 repo to process process_items(these_repos, status_collector, num_jobs=1) + erase_last_line() statuses = status_collector.statuses @@ -147,4 +169,4 @@ def run(args: argparse.Namespace) -> None: # check if we have found all Groups (if any provided) # and if not, throw exception ManifestGroupNotFound - wrs.must_match_all_groups() + wrs.must_match_all_groups(ignore_if_group_not_found=args.ignore_if_group_not_found) diff --git a/tsrc/cli/status.py b/tsrc/cli/status.py index ce320825..77c9622f 100644 --- a/tsrc/cli/status.py +++ b/tsrc/cli/status.py @@ -2,6 +2,7 @@ import argparse from copy import deepcopy +from typing import Union from tsrc.cli import ( add_num_jobs_arg, @@ -15,7 +16,7 @@ from tsrc.groups import GroupNotFound from tsrc.groups_to_find import GroupsToFind from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo -from tsrc.status_endpoint import StatusCollector +from tsrc.status_endpoint import StatusCollector, StatusCollectorLocalOnly from tsrc.status_header import StatusHeader, StatusHeaderDisplayMode from tsrc.utils import erase_last_line @@ -25,24 +26,18 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: parser = subparser.add_parser( - "status", description="Report Status of repositories of current Workspace." + "status", + description="Report Status of repositories of current Workspace. Also report Deep Manifest, Future Manifest and Manifest Marker if presnet.", # noqa: E501 ) add_workspace_arg(parser) add_repos_selection_args(parser) add_num_jobs_arg(parser) - parser.add_argument( - "--verbose", - action="store_true", - help="more verbose if available", - dest="more_verbose", - ) parser.add_argument( "--no-mm", action="store_false", help="do not display Manifest marker", dest="use_manifest_marker", ) - # TODO: 'no_deep_manifest' now uses wrong logic parser.add_argument( "--no-dm", action="store_false", @@ -55,6 +50,12 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: help="do not display Future Manifest", dest="use_future_manifest", ) + parser.add_argument( + "--local-git-only", + action="store_true", + help="do not process anything that will lead to remote connection", + dest="local_git_only", + ) parser.add_argument( "--same-fm", action="store_true", @@ -67,6 +68,18 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: help="do not check for leftover's GIT descriptions", dest="strict_on_git_desc", ) + parser.add_argument( + "--ignore-missing-groups", + action="store_true", + dest="ignore_if_group_not_found", + help="ignore configured group(s) if it is not found in groups defined in manifest. This may be particulary useful when switching Manifest version back when some Groups defined later, was not there yet. In which case we can avoid unecessary Error caused by missing group", # noqa: E501 + ) + parser.add_argument( + "--ignore-missing-group-items", + action="store_true", + dest="ignore_group_item", + help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use", # noqa: E501 + ) parser.set_defaults(run=run) @@ -76,13 +89,21 @@ def run(args: argparse.Namespace) -> None: gtf.found_these(groups_seen) try: - workspace = get_workspace_with_repos(args) + workspace = get_workspace_with_repos( + args, + ignore_if_group_not_found=args.ignore_if_group_not_found, + ignore_group_item=args.ignore_group_item, + ) except GroupNotFound: # try to obtain workspace ignoring group error # if group is found in Deep Manifest or Future Manifest, # do not report GroupNotFound. # if not, than raise exception at the very end - workspace = get_workspace_with_repos(args, ignore_if_group_not_found=True) + workspace = get_workspace_with_repos( + args, + ignore_if_group_not_found=True, + ignore_group_item=args.ignore_group_item, + ) dm = None if args.use_deep_manifest is True: @@ -105,7 +126,15 @@ def run(args: argparse.Namespace) -> None: [StatusHeaderDisplayMode.BRANCH], ) status_header.display() - status_collector = StatusCollector(workspace) + status_collector: Union[StatusCollector, StatusCollectorLocalOnly] + if args.local_git_only is True: + status_collector = StatusCollectorLocalOnly( + workspace, ignore_group_item=args.ignore_group_item + ) + else: + status_collector = StatusCollector( + workspace, ignore_group_item=args.ignore_group_item + ) repos = deepcopy(workspace.repos) @@ -139,4 +168,4 @@ def run(args: argparse.Namespace) -> None: # check if we have found all Groups (if any provided) # and if not, throw exception ManifestGroupNotFound - wrs.must_match_all_groups() + wrs.must_match_all_groups(ignore_if_group_not_found=args.ignore_if_group_not_found) diff --git a/tsrc/cli/sync.py b/tsrc/cli/sync.py index 8e00a644..05194eed 100644 --- a/tsrc/cli/sync.py +++ b/tsrc/cli/sync.py @@ -16,7 +16,10 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: - parser = subparser.add_parser("sync") + parser = subparser.add_parser( + "sync", + description="Synchronize Workspace by current configuration of Manifest (see manifest_url and manifest_branch in config)", # noqa: E501 + ) add_workspace_arg(parser) add_repos_selection_args(parser) parser.set_defaults(update_manifest=True, force=False, correct_branch=True) @@ -39,7 +42,13 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: "--ignore-missing-groups", action="store_true", dest="ignore_if_group_not_found", - help="ignore configured group(s) if it is not found in groups defined in manifest", + help="ignore configured group(s) if it is not found in groups defined in manifest. This may be particulary useful when switching Manifest version back when some Groups defined later, was not there yet. In which case we can avoid unecessary Error caused by missing group", # noqa: E501 + ) + parser.add_argument( + "--ignore-missing-group-items", + action="store_true", + dest="ignore_group_item", + help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use", # noqa: E501 ) parser.add_argument( "--no-correct-branch", @@ -84,13 +93,17 @@ def run(args: argparse.Namespace) -> None: found_groups = list( set(groups).intersection(local_manifest.group_list.groups) ) - workspace.update_config_repo_groups(groups=found_groups) + workspace.update_config_repo_groups( + groups=found_groups, ignore_group_item=args.ignore_group_item + ) report_update_repo_groups = True if update_config_repo_groups is True: if args.ignore_if_group_not_found is True: ignore_if_group_not_found = True - workspace.update_config_repo_groups(groups=found_groups) + workspace.update_config_repo_groups( + groups=found_groups, ignore_group_item=args.ignore_group_item + ) report_update_repo_groups = True if report_update_repo_groups is True: @@ -108,6 +121,7 @@ def run(args: argparse.Namespace) -> None: include_regex=include_regex, exclude_regex=exclude_regex, ignore_if_group_not_found=ignore_if_group_not_found, + ignore_group_item=args.ignore_group_item, ) workspace.clone_missing(num_jobs=num_jobs) @@ -118,5 +132,5 @@ def run(args: argparse.Namespace) -> None: correct_branch=correct_branch, num_jobs=num_jobs, ) - workspace.perform_filesystem_operations() + workspace.perform_filesystem_operations(ignore_group_item=args.ignore_group_item) ui.info_1("Workspace synchronized") diff --git a/tsrc/dump_manifest.py b/tsrc/dump_manifest.py new file mode 100644 index 00000000..a2997711 --- /dev/null +++ b/tsrc/dump_manifest.py @@ -0,0 +1,1154 @@ +""" +dump_manifest + +contains all functions and logic used +in order to obtain or update Manifest dump from current Workspace + +SIDE NOTE: +Q: Why puting so much trouble to going through YAML data on each +single operation of Update? +A: It is due to keep as much original data (including comments) +as possible +""" + +import hashlib +import re +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any, Dict, List, Tuple, Union + +from ruamel.yaml.comments import CommentedMap + +from tsrc.cli import resolve_repos_apply_constraints, resolve_repos_without_workspace +from tsrc.dump_manifest_helper import ManifestRepoItem +from tsrc.groups_and_constraints_data import GroupsAndConstraints +from tsrc.manifest import Manifest +from tsrc.manifest_common_data import ManifestsTypeOfData +from tsrc.repo import Remote, Repo + + +@dataclass(frozen=True) +class ManifestDumpersOptions: + delete_repo: bool = True + add_repo: bool = True # not implemented + update_repo: bool = True # not implemented + + +class ManifestDumper: + def __init__(self) -> None: + pass + + def on_update( + self, + y: Union[Dict, List], + mris: Dict[str, ManifestRepoItem], + opt: ManifestDumpersOptions, + gac: GroupsAndConstraints, + ) -> Tuple[Union[Dict, List], bool]: + """ + if we want to UPDATE existing manifest: + + * Find out if there are some Repos, that should be + renamed (instead of del(old)+add(new)) + + * Apply constraints like Groups, regexes + + Update Repo records in YAML: + * 1st delete Repo(s) that does not exists + * also delete Group item + * and if such Group is empty, delete it + * 2nd update surch Repo(s) that does exists + * 3rd add new Repo(s) that was not updated + + * finaly return something that can be dumped as YAML + """ + is_updated: bool = False + + # get Repos to consider + u_m = Manifest() + # this will print Warning if Group item is not found + u_m.apply_config(y, ignore_on_mtod=ManifestsTypeOfData.DEEP) + repos = resolve_repos_without_workspace(u_m, gac) + + # rename Repos of UPDATE source (early) + tmp_is_updated: bool + tmp_is_updated, repos = self._rename_update_source_based_on_dump_source( + y, mris, repos + ) + is_updated |= tmp_is_updated + + # we now have Repos, that can be work with, so this is the time to: + # * apply Group filtering, * 'include_regex', * 'exclude_regex' + repos, ignored_repos_dests = self._filter_by_groups_and_constraints( + y, gac, repos + ) + + repos_dests: List[str] = [repo.dest for repo in repos] + + # Dump source: Repo(s) + # if data are from Workspace, it already contains Group filtering + # if data comes from RAW dump, it contain all Repos (there cannot be any Groups) + ds_rs: List[str] = list(mris.keys()) + + # UPDATE source: current Manifest's Repo(s) + us_rs: List[str] = [] + self._walk_yaml_get_repos_keys(y, 0, us_rs, False) + + # check if there is some constrain on Repos + is_constrained: bool = self._is_constrained(us_rs, repos) + + # start calculating change lists (Add|Remove|Update Repos) + + a_rs: List[str] # add these Repo(s) + d_rs: List[str] = [] # remove these Repo(s) + u_rs: List[str] # update these Repo(s) + + # check for some constraints applied on Repos + if is_constrained is True: + a_rs = list(set(ds_rs).difference(us_rs).intersection(ignored_repos_dests)) + u_rs = list(set(ds_rs).intersection(us_rs).intersection(repos_dests)) + if opt.delete_repo is True: + d_rs = list(set(us_rs).difference(ds_rs).intersection(repos_dests)) + else: + a_rs = list(set(ds_rs).difference(us_rs)) + u_rs = list(set(ds_rs).intersection(us_rs)) + if opt.delete_repo is True: + d_rs = list(set(us_rs).difference(ds_rs)) + + # 1st A: delete Repo(s) that does not exists + is_updated_tmp: List[bool] = [False] + self._walk_yaml_delete_group_items(y, 0, False, False, d_rs, is_updated_tmp) + is_updated |= is_updated_tmp[0] + + # 1st B: delete also Group Repo items + is_updated_tmp[0] = False + self._walk_yaml_delete_repos_items(y, 0, False, d_rs, is_updated_tmp) + is_updated |= is_updated_tmp[0] + + # 2nd update surch Repo(s) that does exists + is_updated_tmp[0] = False + self._walk_yaml_update_repos_items(y, 0, mris, False, u_rs, is_updated_tmp) + is_updated |= is_updated_tmp[0] + + # 3rd add new Repo(s) that was not updated + is_updated_tmp[0] = False + self._walk_yaml_add_repos_items(y, 0, mris, False, a_rs, is_updated_tmp) + is_updated |= is_updated_tmp[0] + + return y, is_updated + + """ + ================================ + Filter by Groups and constraints + """ + + def _is_constrained(self, us_rs: List[str], repos: List[Repo]) -> bool: + for dest in us_rs: + is_found: bool = False + for repo in repos: + if repo.dest == dest: + is_found = True + break + if is_found is False: + return True + return False + + def _filter_by_groups_and_constraints( + self, + y: Union[Dict, List], + gac: GroupsAndConstraints, + repos: List[Repo], + ) -> Tuple[List[Repo], List[str]]: + ignored_repos_dests: List[str] = [] + if gac.groups: + o_repos: List[Repo] = [] + u_m = Manifest() + # we need new current Manifest with renamed Repos and Group items + u_m.apply_config(y, ignore_on_mtod=ManifestsTypeOfData.DEEP_ON_UPDATE) + m_groups: List[str] = [] + if u_m.group_list: + for gr in u_m.group_list.groups: + if gr in gac.groups: + m_groups.append(gr) + for e in u_m.group_list.get_elements(m_groups): + o_repos.append(u_m.get_repo(e)) + repos = o_repos + if u_m.group_list and u_m.group_list.missing_elements: + for mi in u_m.group_list.missing_elements: + for k_r_d, i_r_d in mi.items(): + i_r_d_value = self._ready_ignored_repos_dests(k_r_d, gac, i_r_d) + if i_r_d_value: + ignored_repos_dests.append(i_r_d_value) + + repos = resolve_repos_apply_constraints(repos, gac) + + return repos, ignored_repos_dests + + def _ready_ignored_repos_dests( + self, + k_r_d: str, + gac: GroupsAndConstraints, + i_r_d: str, + ) -> str: + if gac.groups and k_r_d in gac.groups: + if ( + ( + gac.include_regex and re.search(gac.include_regex, i_r_d) + ) # noqa: W503 + or not gac.include_regex # noqa: W503 + ) and ( + ( + gac.exclude_regex + and not re.search(gac.exclude_regex, i_r_d) # noqa: W503 + ) + or not gac.exclude_regex # noqa: W503 + ): + return i_r_d + return "" + + """ + ================================= + Prepare data for Renaming process + """ + + def _rename_update_source_based_on_dump_source( + self, + y: Union[Dict, List], + mris: Dict[str, ManifestRepoItem], + repos: List[Repo], # UPDATE Repos (from Manifest) + ) -> Tuple[bool, List[Repo]]: + is_updated: bool = False + + dump_urls_dict: OrderedDict[str, str] = OrderedDict() + rename_repo_dict: OrderedDict[str, str] = OrderedDict() + + # create URL dictionary + dump_urls_dict = self._get_dump_url_dict(mris) + + # create 1st shot of dictionary for renaming + rename_repo_dict = self._get_rename_repo_dict(dump_urls_dict, repos) + + # get rid of colisions + rename_repo_dict_pre: Dict[str, str] = {} + rename_repo_dict_post: Dict[str, str] = {} + rename_repo_dict_pre, rename_repo_dict_post = ( + self._get_pre_and_post_rename_repo_dict(dump_urls_dict, rename_repo_dict) + ) + + # Rename repositories entries + is_updated_tmp: List[bool] = [False] + self._walk_yaml_rename_repos_items( + y, 0, False, rename_repo_dict_pre, repos, is_updated_tmp + ) + is_updated |= is_updated_tmp[0] + is_updated_tmp = [False] + self._walk_yaml_rename_repos_items( + y, 0, False, rename_repo_dict_post, repos, is_updated_tmp + ) + is_updated |= is_updated_tmp[0] + + # rename on Group's Repo items + is_updated_tmp = [False] + self._walk_yaml_rename_group_items( + y, 0, False, False, rename_repo_dict_pre, is_updated_tmp + ) + is_updated |= is_updated_tmp[0] + is_updated_tmp = [False] + self._walk_yaml_rename_group_items( + y, 0, False, False, rename_repo_dict_post, is_updated_tmp + ) + is_updated |= is_updated_tmp[0] + + return is_updated, repos + + def _get_dump_url_dict( + self, mris: Dict[str, ManifestRepoItem] + ) -> "OrderedDict[str, str]": + dump_urls_dict: OrderedDict[str, str] = OrderedDict() + for k, v in mris.items(): + if v.remotes: + for remote in v.remotes: + dump_urls_dict[remote.url] = k + return dump_urls_dict + + def _get_rename_repo_dict( + self, + dump_urls_dict: "OrderedDict[str, str]", + repos: List[Repo], + ) -> "OrderedDict[str, str]": + rename_repo_dict: OrderedDict[str, str] = OrderedDict() + for repo in repos: + if repo.remotes: + for remote in repo.remotes: + if remote.url in dump_urls_dict: + if ( + repo.dest != dump_urls_dict[remote.url] + and repo.dest not in rename_repo_dict # noqa: W503 + ): + rename_repo_dict[repo.dest] = dump_urls_dict[remote.url] + return rename_repo_dict + + def _get_pre_and_post_rename_repo_dict( + self, + dump_urls_dict: "OrderedDict[str, str]", + rename_repo_dict: "OrderedDict[str, str]", + ) -> Tuple[Dict[str, str], Dict[str, str]]: + rename_repo_dict_pre: Dict[str, str] = {} + rename_repo_dict_post: Dict[str, str] = {} + for key, val in rename_repo_dict.items(): + unique_key = key + offset: int = 0 + # while unique_key in rename_repo_dict.values(): + while unique_key in dump_urls_dict.values(): + # create unique sumplement for key + unique_key = val + "-" + self._get_sha1_plus(val, offset)[:7] + offset += 1 + if key != unique_key: + # repeling colision + rename_repo_dict_pre[key] = unique_key + rename_repo_dict_post[unique_key] = val + else: + # when no colision, add to the 'post' + rename_repo_dict_post[key] = val + return rename_repo_dict_pre, rename_repo_dict_post + + def _get_sha1_plus(self, name: str, p: int = 0) -> str: + str_sha1 = hashlib.sha1() + name = name + str(p) + str_sha1.update(name.encode("utf-8")) + return str_sha1.hexdigest() + + """ + ============================================= + Renaming Repositories entries into the Manifest + by walking through YAML file. + """ + + def _rename_repos_based_on_rrd( + self, + y: Dict, + rrd: Dict[str, str], # rename Repo Dict + repos: List[Repo], + is_updated: List[bool], + ) -> bool: + ret_updated: bool = is_updated[0] + + if "dest" in y and y["dest"] in rrd.keys(): + dest = y["dest"] + if y["dest"] != rrd[dest]: + ret_updated |= True + for repo in repos: + if repo.dest == y["dest"]: + # rename in Repo + repo.rename_dest(rrd[dest]) + # rename in YAML data + y["dest"] = rrd[dest] + + return ret_updated + + def _walk_yaml_rename_repos_items_on_dict( + self, + y: Union[Dict, List], + level: int, + on_repos: bool, + rrd: Dict[str, str], # rename Repo Dict + repos: List[Repo], + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_rename_repos_items( + y[key], level + 1, on_repos, rrd, repos, is_updated + ) + return ready_return + + def _walk_yaml_rename_repos_items( + self, + y: Union[Dict, List], + level: int, + on_repos: bool, + rrd: Dict[str, str], # rename Repo Dict + repos: List[Repo], + is_updated: List[bool], + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_rename_repos_items_on_dict( + y, level, on_repos, rrd, repos, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + + for index, item in enumerate(y): + if on_repos is True and isinstance(item, dict) and "dest" in item: + + # Rename it here + is_updated[0] |= self._rename_repos_based_on_rrd( + y[index], rrd, repos, is_updated + ) + + self._walk_yaml_rename_repos_items( + item, level, False, rrd, repos, is_updated + ) + + """ + ============================================= + Renaming Group items of the Manifest + by walking through YAML file. + """ + + def _walk_yaml_rename_group_items_on_dict( + self, + y: Union[Dict, List], + level: int, + on_groups: bool, + on_g_r: bool, + rrd: Dict[str, str], # rename Repo Dict + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if on_groups is True and level == 2 and key == "repos": + on_g_r = True + elif level == 1: + on_g_r = False + + if key == "groups" and level == 0: + on_groups = True + elif level == 0: + on_groups = False + on_g_r = False + self._walk_yaml_rename_group_items( + y[key], level + 1, on_groups, on_g_r, rrd, is_updated + ) + return ready_return + + def _walk_yaml_rename_group_items( + self, + y: Union[Dict, List], + level: int, + on_groups: bool, + on_g_r: bool, + rrd: Dict[str, str], # rename Repo Dict + is_updated: List[bool], + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_rename_group_items_on_dict( + y, level, on_groups, on_g_r, rrd, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + + elif isinstance(y, list): + for index, item in enumerate(y): + if on_groups is True and on_g_r is True: + if item in rrd.keys(): + # Rename it here + y[index] = rrd[item] # this is it + + self._walk_yaml_rename_group_items( + item, level, False, False, rrd, is_updated + ) + + """ + ============================================== + Obtaining 'dest' of Repositories from Manifest + by walking through YAML file. + """ + + def _walk_yaml_get_repos_keys_on_dict( + self, y: Union[Dict, List], level: int, repos_dest: List[str], on_repos: bool + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_get_repos_keys(y[key], level + 1, repos_dest, on_repos) + return ready_return + + def _walk_yaml_get_repos_keys( + self, + y: Union[Dict, List], + level: int, + repos_dest: List[str], + on_repos: bool, + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_get_repos_keys_on_dict( + y, level, repos_dest, on_repos + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + + for item in y: + if on_repos is True and isinstance(item, dict) and "dest" in item: + + # we have found 'dest' (in Repo), add it then + repos_dest.append(item["dest"]) + + self._walk_yaml_get_repos_keys(item, level, repos_dest, on_repos) + else: + self._walk_yaml_get_repos_keys(item, level, repos_dest, False) + + """ + ============================================= + Adding Repositories entries into the Manifest + by walking through YAML file. + """ + # TODO: add comments to YAML + + def _add_repos_based_on_mris( + self, + y: List, + a_rs: List[str], + mris: Dict[str, ManifestRepoItem], + is_updated: List[bool], + ) -> bool: + ret_updated: bool = is_updated[0] + for a_r in a_rs: + if mris[a_r]: + mri = mris[a_r] + + rr = CommentedMap() + rr["dest"] = a_r + if mri.remotes: + if len(mri.remotes) == 1 and mri.remotes[0].name == "origin": + rr["url"] = mri.clone_url + else: + rr["remotes"] = self._do_create_on_remotes(mri.remotes) + if mri.ignore_submodules and mri.ignore_submodules is True: + rr["ignore_submodules"] = True + if mri.branch: + rr["branch"] = mri.branch + if mri.tag: + rr["tag"] = mri.tag + if not mri.branch and not mri.tag and mri.sha1: + rr["sha1"] = mri.sha1 + + # TODO: add comment in form of '\n' just to better separate Repos + y.append(rr) + ret_updated = True + + return ret_updated + + def _walk_yaml_add_repos_items_on_dict( + self, + y: Union[Dict, List], + level: int, + mris: Dict[str, ManifestRepoItem], + on_repos: bool, + a_rs: List[str], # (to) add: (list of) Repos + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_add_repos_items( + y[key], level + 1, mris, on_repos, a_rs, is_updated + ) + return ready_return + + def _walk_yaml_add_repos_items( + self, + y: Union[Dict, List], + level: int, + mris: Dict[str, ManifestRepoItem], + on_repos: bool, + a_rs: List[str], # (to) add (from) Repos + is_updated: List[bool], + dest: Union[str, None] = None, + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_add_repos_items_on_dict( + y, level, mris, on_repos, a_rs, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + + go_add: bool = False + if len(y) > 0: + for item in y: + if on_repos is True and isinstance(item, dict) and "dest" in item: + + go_add = True + + self._walk_yaml_add_repos_items( + item, level, mris, on_repos, a_rs, is_updated, item["dest"] + ) + else: + self._walk_yaml_add_repos_items( + item, level, mris, False, a_rs, is_updated + ) + else: # there are no items at all, therefore we are at the end + go_add = True + + if go_add is True and on_repos is True: + # add all Repos in here + is_updated[0] |= self._add_repos_based_on_mris( + y, a_rs, mris, is_updated + ) + + """ + =============================================== + Updating Repositories entries into the Manifest + by walking through YAML file. + """ + + def _delete_on_update_on_items_on_repo( + self, + y: Dict, + d_is: List[str], # delete (these) items + ) -> bool: + ret_updated: bool = False + for d_i in d_is: + if y[d_i]: + del y[d_i] + ret_updated = True + return ret_updated + + def _update_on_update_on_items_on_repo( + self, + y: Dict, + mri: ManifestRepoItem, + u_is: List[str], # update (these) items + ) -> bool: + ret_updated: bool = False + for u_i in u_is: + if ( + u_i == "branch" + and mri.branch # noqa: W503 + and y[u_i] != mri.branch # noqa: W503 + ): + y[u_i] = mri.branch + ret_updated = True + + if u_i == "tag" and mri.tag and y[u_i] != mri.tag: + y[u_i] = mri.tag + ret_updated = True + + # 'sha1' is only updated on special case + if ( + u_i == "sha1" + and not mri.branch # noqa: W503 + and not mri.tag # noqa: W503 + and mri.sha1 # noqa: W503 + and y[u_i] != mri.sha1 # noqa: W503 + ): + y[u_i] = mri.sha1 + ret_updated = True + return ret_updated + + def _add_on_update_on_items_on_repo( + self, + y: Dict, + mri: ManifestRepoItem, + a_is: List[str], # add (these) items + ) -> bool: + ret_updated: bool = False + for a_i in a_is: + if a_i == "branch": + y[a_i] = mri.branch + ret_updated = True + if a_i == "tag": + y[a_i] = mri.tag + ret_updated = True + if a_i == "sha1": + y[a_i] = mri.sha1 + ret_updated = True + return ret_updated + + def _remotes_on_update_on_items_on_repo_need_update( + self, + y: Dict, + mri: ManifestRepoItem, + ) -> bool: + # return True if update is needed + need_update: bool = False + # check case when we have "remotes" in Manifest + if "remotes" in y and mri.remotes: + if len(y["remotes"]) != len(mri.remotes): + need_update = True + else: + lyr: List[Remote] = [] + for yr in y["remotes"]: + if "name" in yr and "url" in yr: + lyr.append(Remote(name=yr["name"], url=yr["url"])) + + for remote in lyr + mri.remotes: + if remote not in lyr or remote not in mri.remotes: + need_update = True + break + + # check case when we have "url" in Manifest + elif "url" in y: + if (mri.remotes and len(mri.remotes) > 1) or y["url"] != mri.clone_url: + need_update = True + + # check if we need to add remotes to Manifest + else: + if mri.remotes: + need_update = True + + return need_update + + def _remotes_on_update_on_items_on_repo( + self, + y: Dict, + mri: ManifestRepoItem, + ) -> bool: + # 1st: check if need to update + need_update: bool = self._remotes_on_update_on_items_on_repo_need_update(y, mri) + + # 2nd: do actual update only when needed + if need_update is True: + if "url" in y and mri.remotes: + if len(mri.remotes) == 1 and mri.remotes[0].name == "origin": + # just simple update + y["url"] = mri.clone_url + else: + del y["url"] + y["remotes"] = self._do_create_on_remotes(mri.remotes) + + elif "remotes" in y and mri.remotes: + # check if we may want to use plain "url" instead of "remotes" + if len(mri.remotes) == 1 and mri.remotes[0].name == "origin": + del y["remotes"] + y["url"] = mri.clone_url + else: + y["remotes"] = self._do_create_on_remotes(mri.remotes) + else: + # we need to create such record + if mri.remotes: + if len(mri.remotes) == 1 and mri.remotes[0].name == "origin": + y["url"] = mri.clone_url + else: + y["remotes"] = self._do_create_on_remotes(mri.remotes) + + return need_update + + def _update_on_items_on_repo( + self, + y: Dict, + mri: ManifestRepoItem, + ) -> bool: + ret_updated: bool = False + + c_item: List[str] = [] # current item (only those we want to consider) + for key in y.keys(): + if key == "branch" or key == "tag" or key == "sha1": + c_item.append(key) + + # states items (only those we want to consider) + s_item: List[str] = [] + for key in ["branch", "tag"]: # "sha1" should used by request + if (key == "branch" and mri.branch) or (key == "tag" and mri.tag): + s_item.append(key) + if not s_item: + if mri.sha1: + s_item.append("sha1") + + # add these on item + a_is: List[str] = list(set(s_item).difference(c_item)) + + # remove these on item + d_is: List[str] = list(set(c_item).difference(s_item)) + + # update these on item + u_is: List[str] = list(set(s_item).intersection(c_item)) + + # perform action(s) and check if it gets updated + ret_updated |= self._delete_on_update_on_items_on_repo(y, d_is) + ret_updated |= self._update_on_update_on_items_on_repo(y, mri, u_is) + ret_updated |= self._add_on_update_on_items_on_repo(y, mri, a_is) + + ret_updated |= self._remotes_on_update_on_items_on_repo(y, mri) + + return ret_updated + + def _update_repos_based_on_mris( + self, + y: Dict, + u_rs: List[str], + mris: Dict[str, ManifestRepoItem], + is_updated: List[bool], + ) -> bool: + ret_updated: bool = is_updated[0] + + if "dest" in y and y["dest"] in u_rs: + dest = y["dest"] + if mris[dest]: + ret_updated |= self._update_on_items_on_repo(y, mris[dest]) + + return ret_updated + + def _walk_yaml_update_repos_items_on_dict( + self, + y: Union[Dict, List], + level: int, + mris: Dict[str, ManifestRepoItem], + on_repos: bool, + u_rs: List[str], # update (list of) Repos + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_update_repos_items( + y[key], level + 1, mris, on_repos, u_rs, is_updated + ) + return ready_return + + def _walk_yaml_update_repos_items( + self, + y: Union[Dict, List], + level: int, + mris: Dict[str, ManifestRepoItem], + on_repos: bool, + u_rs: List[str], # update (this list of) Repos + is_updated: List[bool], + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_update_repos_items_on_dict( + y, level, mris, on_repos, u_rs, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + + for index, item in enumerate(y): + if on_repos is True and isinstance(item, dict) and "dest" in item: + + # Update it here + is_updated[0] |= self._update_repos_based_on_mris( + y[index], u_rs, mris, is_updated + ) + + self._walk_yaml_update_repos_items( + item, level, mris, False, u_rs, is_updated + ) + + """ + =========================================== + Deleting Repositories entries from Manifest + by walking through YAML file. + """ + + def _walk_yaml_delete_group_items_on_dict( + self, + y: Union[Dict, List], + level: int, + on_groups: bool, + on_g_r: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if on_groups is True and level == 2 and key == "repos": + on_g_r = True + elif level == 1: + on_g_r = False + + if key == "groups" and level == 0: + on_groups = True + elif level == 0: + on_groups = False + on_g_r = False + self._walk_yaml_delete_group_items( + y[key], level + 1, on_groups, on_g_r, d_rs, is_updated + ) + return ready_return + + def _walk_yaml_delete_group_items__on_list( + self, + y: Union[Dict, List], + level: int, + on_groups: bool, + on_g_r: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + ) -> None: + go_dels: List[int] = [] + for index, item in enumerate(y): + if on_groups is True and on_g_r is True: + if item in d_rs: + go_dels.append(index) + self._walk_yaml_delete_group_items( + item, level, False, False, d_rs, is_updated + ) + + if go_dels: + # traverse backwards in order not to hurt earlier indexes + for gd in reversed(go_dels): + del y[gd] + is_updated[0] = True + + def _walk_yaml_delete_group_items( + self, + y: Union[Dict, List], + level: int, + on_groups: bool, + on_g_r: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_delete_group_items_on_dict( + y, level, on_groups, on_g_r, d_rs, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + + elif isinstance(y, list): + self._walk_yaml_delete_group_items__on_list( + y, level, on_groups, on_g_r, d_rs, is_updated + ) + + """ + ---- Delete Repos items + """ + + def _walk_yaml_delete_repos_items_on_dict( + self, + y: Union[Dict, List], + level: int, + on_repos: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_delete_repos_items( + y[key], level + 1, on_repos, d_rs, is_updated + ) + return ready_return + + def _walk_yaml_delete_repos_items__on_list( + self, + y: Union[Dict, List], + level: int, + on_repos: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + dest: Union[str, None] = None, + ) -> None: + go_dels: List[int] = [] + for index, item in enumerate(y): + if on_repos is True and isinstance(item, dict) and "dest" in item: + + # identify item + if item["dest"] in d_rs: + go_dels.append(index) + + self._walk_yaml_delete_repos_items( + item, + level, + on_repos, + d_rs, + is_updated, + item["dest"], + ) + else: + self._walk_yaml_delete_repos_items(item, level, False, d_rs, is_updated) + + if go_dels: + # traverse backwards in order not to hurt earlier indexes + for gd in reversed(go_dels): + del y[gd] + is_updated[0] = True + + def _walk_yaml_delete_repos_items( + self, + y: Union[Dict, List], + level: int, + on_repos: bool, + d_rs: List[str], # (to) delete: (list of) Repos identified by 'dest' + is_updated: List[bool], + dest: Union[str, None] = None, + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_delete_repos_items_on_dict( + y, level, on_repos, d_rs, is_updated + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + self._walk_yaml_delete_repos_items__on_list( + y, level, on_repos, d_rs, is_updated, dest + ) + + """ + ========================================== + Creating by filling data to YAML structure + from ManifestRepoItem + """ + + def _do_create_on_remotes(self, remotes: List[Remote]) -> List[Dict[str, str]]: + cs: List[Dict[str, str]] = [] # --> CommentedSeq + cm: Dict[str, str] = {} # --> CommentedMap + + for sr in remotes: + # make sure that "origin" will be first + # this is important to keep default remote first + if sr.name == "origin": + cm = {} + cm["name"] = sr.name + cm["url"] = sr.url + cs.append(cm) + for sr in remotes: + if sr.name != "origin": + cm = {} + cm["name"] = sr.name + cm["url"] = sr.url + cs.append(cm) + return cs + + def do_create( + self, + mris: Dict[str, ManifestRepoItem], + ) -> Dict: + y: Dict[str, Any] = {"repos": []} + + # NOTE: nothing we can do about Group + + for dest, items in mris.items(): + + rr = CommentedMap() + rr["dest"] = dest + if items.remotes: + if len(items.remotes) == 1 and items.remotes[0].name == "origin": + rr["url"] = items.clone_url + else: + rr["remotes"] = self._do_create_on_remotes(items.remotes) + if items.ignore_submodules is True: + rr["ignore_submodules"] = True + if items.branch: + rr["branch"] = items.branch + if items.tag: + rr["tag"] = items.tag + if not items.branch and not items.tag and items.sha1: + rr["sha1"] = items.sha1 + + y["repos"].append(rr) + return y + + """ + =========================================== + Check YAML data structure if all Repos have + at leas one remote. + """ + + def some_remote_is_missing(self, yy: Union[Dict, List, None]) -> bool: + tmp_is_remote: List[bool] = [True] + if yy: + self._walk_yaml_check_remote_in_repo(yy, 0, tmp_is_remote, False) + return not tmp_is_remote[0] + + def _walk_yaml_check_remote_in_repo_on_dict( + self, y: Union[Dict, List], level: int, is_remotes: List[bool], on_repos: bool + ) -> bool: + ready_return = True + for _, key in enumerate(y): + if isinstance(key, tuple): + ready_return = False + if isinstance(key, str): + if key == "repos" and level == 0: + on_repos = True + elif level == 0: + on_repos = False + self._walk_yaml_check_remote_in_repo( + y[key], level + 1, is_remotes, on_repos + ) + return ready_return + + def _walk_yaml_check_remote_in_repo( + self, + y: Union[Dict, List], + level: int, + is_remotes: List[bool], + on_repos: bool, + ) -> None: + if isinstance(y, dict): + ready_return = self._walk_yaml_check_remote_in_repo_on_dict( + y, level, is_remotes, on_repos + ) + if ready_return is True: + return + items = list(y.items()) + for key in items: + y.pop(key) + elif isinstance(y, list): + + for item in y: + if on_repos is True and isinstance(item, dict) and "dest" in item: + + if not ("remote" in item or "url" in item): + is_remotes[0] = False + + # we have found 'dest' (in Repo), add it then + + self._walk_yaml_check_remote_in_repo( + item, level, is_remotes, on_repos + ) + else: + self._walk_yaml_check_remote_in_repo(item, level, is_remotes, False) diff --git a/tsrc/dump_manifest_args.py b/tsrc/dump_manifest_args.py new file mode 100644 index 00000000..865d0b8f --- /dev/null +++ b/tsrc/dump_manifest_args.py @@ -0,0 +1,125 @@ +import argparse +import os +from copy import deepcopy +from pathlib import Path +from typing import List + +import cli_ui as ui + +from tsrc.dump_manifest import ManifestDumpersOptions +from tsrc.dump_manifest_args_data import ( + DumpManifestOperationDetails, + FinalOutputModeFlag, +) +from tsrc.dump_manifest_args_final_output import FinalOutput +from tsrc.dump_manifest_args_source_mode import SourceMode, SourceModeEnum +from tsrc.dump_manifest_args_update_source import UpdateSource +from tsrc.groups_and_constraints_data import ( + GroupsAndConstraints, + get_group_and_constraints_data, +) + + +class DumpManifestArgs: + """ + atempt to separate logic around 'args' and its handling outside of + DumpManifestsCMDLogic + """ + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + + # from args, get Group and constraints data + self.gac: GroupsAndConstraints = get_group_and_constraints_data(args) + + # some (local) helpers + self.mdo: ManifestDumpersOptions + self.any_update = args.do_update or bool(args.update_on) + if self.any_update is True: + self.mdo = ManifestDumpersOptions(delete_repo=not args.no_repo_delete) + else: + self.mdo = ManifestDumpersOptions() + + # this will be returned when no Exception is hit + self.dmod = DumpManifestOperationDetails() + + # ready source MODE + self.s_m = SourceMode(args, self.dmod) + self.dmod, self.args = self.s_m.get_source_mode_and_path() + + # take care of UPDATE source + self.u_s = UpdateSource(args, self.dmod) + self.dmod = self.u_s.get_update_source_and_path() + + # take care of Final Output Mode Flag and All Paths + self.f_o = FinalOutput(args, self.dmod) + self.dmod = self.f_o.get_final_output_modes_and_paths() + + # take care of default situation: get mode and path + self.dmod = self._check_default_mode_and_path() + + # take care of Warning of common purpose + self._take_care_of_common_warnings() + + def _check_default_mode_and_path(self) -> DumpManifestOperationDetails: + # use default only if COMMON PATH will not be calculated + if ( + not self.dmod.final_output_mode + and self.dmod.source_mode != SourceModeEnum.RAW_DUMP # noqa: W503 + ): + if self.dmod.final_output_path_list.default_path.is_file() is True: + if self.args.use_force is False: + raise Exception( + f"such file '{self.dmod.final_output_path_list.default_path}' already exists, use '--force' to overwrite it" # noqa: E501 + ) + else: + + # when there is '--force' allow overwrite of default file + self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) + + else: + + # by default, new file will be created + self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) + + return self.dmod + + def _take_care_of_common_warnings(self) -> None: + if self.args.save_to and self.args.just_preview is True: + ui.warning("'SAVE_TO' path will be ignored when using '--preview'") + + if self.args.save_to and self.args.update_on: + ui.warning("'SAVE_TO' path will be ignored when using '--update-on'") + + if self.any_update is True and self.args.just_preview is True: + ui.warning("When in preview mode, no actual update will be made") + + def consider_common_path( + self, common_path: List[str] + ) -> DumpManifestOperationDetails: + + # verify if it is ok to continue + tmp_save_file = deepcopy(common_path) + tmp_default_file = str(self.dmod.final_output_path_list.default_path).split( + os.sep + ) + tmp_save_file += tmp_default_file + tmp_save_file_path = Path(os.sep.join(tmp_save_file)) + if self.args.raw_dump_path and not self.args.save_to: # noqa: W503 + grab_save_path = tmp_save_file_path + if grab_save_path.is_file(): + if self.args.use_force is True: + self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) + else: + raise Exception( + f"Such file '{grab_save_path}' already exists, use '--force' if you want to overwrite it" # noqa: E501 + ) + else: + # create new file only if we are not updating + if FinalOutputModeFlag.UPDATE not in self.dmod.final_output_mode: + self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) + + # save data + self.dmod.final_output_path_list.common_path = grab_save_path + + return self.dmod diff --git a/tsrc/dump_manifest_args_data.py b/tsrc/dump_manifest_args_data.py new file mode 100644 index 00000000..82c8c30e --- /dev/null +++ b/tsrc/dump_manifest_args_data.py @@ -0,0 +1,123 @@ +from dataclasses import dataclass, field, fields +from enum import Enum, Flag, unique +from pathlib import Path +from typing import Any, List, Optional, Union + +from tsrc.workspace import Workspace + + +@unique +class SourceModeEnum(Enum): + NONE = 0 + RAW_DUMP = 1 + WORKSPACE_DUMP = 2 + YAML_FILE = 3 # not implemented + + +@unique +class UpdateSourceEnum(Enum): + NONE = 0 + FILE = 1 + DEEP_MANIFEST = 2 + + +class FinalOutputModeFlag(Flag): + NONE = 0 + PREVIEW = 1 + NEW = 2 # use: 'destination_path' + # if there is no NEW, that means we are using same file for + # output as we are using for update. thus there cannot be a OVERWRITE + UPDATE = 4 # use: 'update_path' + OVERWRITE = 8 # must use force + + +@dataclass +class FinalOutputAllPaths: + """ + There may not be clear what output path will be used in the end, + therefore we should keep them all and take one at the very end + """ + + default_path: Path = Path("manifest.yml") # use when there is no other + update_on_path: Optional[Path] = None # when using update|update_on + save_to_path: Optional[Path] = None # whenever there should be a new file + common_path: Optional[Path] = None # when on RAW mode, this gets calculated + + def __init__(self, **kwargs: Any) -> None: + # only set those that are present + names = {f.name for f in fields(self)} + for key, value in kwargs.items(): + if key in names: + setattr(self, key, value) + + def clean_all_paths(self) -> None: + for i in dir(self): + if isinstance(getattr(self, i), Path) and i != "default_path": + setattr(self, i, None) + + +@dataclass +class DumpManifestOperationDetails: + """ + Contains all data that should be used + initially for 'dump-manifest' command + + There are cases however that will require + to further checks during execution time + (for example when COMMON PATH will be in place) + """ + + # SOURCE of data (must be set) + source_mode = SourceModeEnum.NONE + source_path: Optional[Path] = None + + # UPDATE source (optional) + update_source = UpdateSourceEnum.NONE + update_source_path: Optional[Path] = None + + # FINAL OUTPUT MODE (must be determined) + final_output_mode: List[FinalOutputModeFlag] = field(default_factory=list) + final_output_path_list = FinalOutputAllPaths() + + # helpers to be used later + workspace: Optional[Workspace] = None + + def __init__(self, **kwargs: Any) -> None: + # fix missing value + if "final_output_mode" not in fields(self): + self.final_output_mode: List[FinalOutputModeFlag] = [] + + # only set those that are present + names = {f.name for f in fields(self)} + for key, value in kwargs.items(): + if key in names: + setattr(self, key, value) + + def clean(self) -> None: + self.final_output_path_list.clean_all_paths() + self.final_output_mode = [] + + # ---------- + # 'get_path' - section + # ---------- + # it is used as for 'final_output_path' only + + def get_path_for_new(self) -> Union[Path, None]: + if self.final_output_path_list.save_to_path: + return self.final_output_path_list.save_to_path + if self.final_output_path_list.common_path: + return self.final_output_path_list.common_path + if self.final_output_path_list.default_path: + return self.final_output_path_list.default_path + + return None + + def get_path_for_update(self) -> Union[Path, None]: + if self.final_output_path_list.save_to_path: + return self.final_output_path_list.save_to_path + if self.final_output_path_list.update_on_path: + return self.final_output_path_list.update_on_path + if self.final_output_path_list.common_path: + return self.final_output_path_list.common_path + + return None diff --git a/tsrc/dump_manifest_args_final_output.py b/tsrc/dump_manifest_args_final_output.py new file mode 100644 index 00000000..e2e23120 --- /dev/null +++ b/tsrc/dump_manifest_args_final_output.py @@ -0,0 +1,65 @@ +# FinalOutput (ModeFlag|AllPaths) +import argparse +import os + +from tsrc.dump_manifest_args_data import ( + DumpManifestOperationDetails, + FinalOutputModeFlag, +) + + +class FinalOutput: + + def __init__( + self, args: argparse.Namespace, dmod: DumpManifestOperationDetails + ) -> None: + self.args = args + self.dmod = dmod + + def get_final_output_modes_and_paths(self) -> DumpManifestOperationDetails: + + # on '--preview' + self._take_care_of__preview() + + # on '--save_to' + self._take_care_of__save_to() + + return self.dmod + + def _take_care_of__preview(self) -> None: + if self.args.just_preview is True: + # no output path in this case + self.dmod.final_output_mode.append(FinalOutputModeFlag.PREVIEW) + + def _take_care_of__save_to(self) -> None: + if self.args.save_to: + if self.args.save_to.is_dir() is True: + self.args.save_to = self.args.save_to / "manifest.yml" + elif ( + os.path.dirname(self.args.save_to) + and os.path.isdir(os.path.dirname(self.args.save_to)) # noqa: W503 + is False # noqa: W503 + ): + raise Exception( + f"'SAVE_TO' directory structure must exists, however '{os.path.dirname(self.args.save_to)}' does not" # noqa: E501 + ) + if self.args.save_to.is_file() is True: + if ( + self.args.use_force is False + and FinalOutputModeFlag.PREVIEW # noqa: W503 + not in self.dmod.final_output_mode + ): + raise Exception( + f"'SAVE_TO' file exist, use '--force' to overwrite existing file, or use '--update-on {self.args.save_to}' instead" # noqa: E501 + ) + else: + + # set data only in regard of output + self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) + self.dmod.final_output_path_list.save_to_path = self.args.save_to + + else: + + # set data only in regard of output + self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) + self.dmod.final_output_path_list.save_to_path = self.args.save_to diff --git a/tsrc/dump_manifest_args_source_mode.py b/tsrc/dump_manifest_args_source_mode.py new file mode 100644 index 00000000..1bd75d3d --- /dev/null +++ b/tsrc/dump_manifest_args_source_mode.py @@ -0,0 +1,57 @@ +import argparse +import os +from pathlib import Path +from typing import Tuple + +from tsrc.cli import get_workspace_with_repos +from tsrc.dump_manifest_args_data import DumpManifestOperationDetails, SourceModeEnum + + +class SourceMode: + + def __init__( + self, args: argparse.Namespace, dmod: DumpManifestOperationDetails + ) -> None: + self.args = args + self.dmod = dmod + + def get_source_mode_and_path( + self, + ) -> Tuple[DumpManifestOperationDetails, argparse.Namespace]: + + self._respect_workspace_path() + + self._decide_source_mode() + + self._get_workspace_if_needed() + + return self.dmod, self.args + + def _respect_workspace_path(self) -> None: + # when Workspace path is provided by '-w', we have to consider + # it as root path when relative path is provided for RAW dump + if ( + self.args.raw_dump_path + and self.args.workspace_path # noqa: W503 + and not os.path.isabs(self.args.raw_dump_path) # noqa: W503 + ): + self.args.raw_dump_path = Path( + os.path.join(self.args.workspace_path, self.args.raw_dump_path) + ) + + def _decide_source_mode(self) -> None: + if self.args.raw_dump_path: + self.dmod.source_mode = SourceModeEnum.RAW_DUMP + self.dmod.source_path = self.args.raw_dump_path + else: + # right now there are no more 'Source MODEs' implemented + # therefore any other then RAW MODE is Workspace MODE + self.dmod.source_mode = SourceModeEnum.WORKSPACE_DUMP + + def _get_workspace_if_needed(self) -> None: + # determine if Workspace is required + if not self.args.raw_dump_path or ( + self.args.raw_dump_path and self.args.do_update is True + ): + # it will throw Error if there is no Workspace + self.dmod.workspace = get_workspace_with_repos(self.args) diff --git a/tsrc/dump_manifest_args_update_source.py b/tsrc/dump_manifest_args_update_source.py new file mode 100644 index 00000000..f086a0a4 --- /dev/null +++ b/tsrc/dump_manifest_args_update_source.py @@ -0,0 +1,144 @@ +import argparse +import os + +import cli_ui as ui + +from tsrc.dump_manifest_args_data import ( + DumpManifestOperationDetails, + FinalOutputModeFlag, + UpdateSourceEnum, +) +from tsrc.dump_manifest_args_source_mode import SourceModeEnum +from tsrc.git import GitStatus, run_git_captured +from tsrc.groups_to_find import GroupsToFind +from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo + + +class UpdateSource: + + def __init__( + self, args: argparse.Namespace, dmod: DumpManifestOperationDetails + ) -> None: + self.args = args + self.dmod = dmod + + def get_update_source_and_path(self) -> DumpManifestOperationDetails: + + self._possible_mistmatch_on_dump_path() + + self._allow_only_1_update_on_a_time() + + # decide on mode + if self.args.do_update is True: + # obtaining data - part + if self.dmod.source_mode == SourceModeEnum.WORKSPACE_DUMP or ( + self.dmod.source_mode == SourceModeEnum.RAW_DUMP + ): + self._get_dm_load_path() + elif self.args.update_on: + # test if such file exists + self._update_on_file_must_exist() + + # we have data ready - no operation after this block in here + self.dmod.update_source = UpdateSourceEnum.FILE + self.dmod.update_source_path = self.args.update_on + self.dmod.final_output_path_list.update_on_path = self.args.update_on + self.dmod.final_output_mode.append(FinalOutputModeFlag.UPDATE) + + return self.dmod + + def _possible_mistmatch_on_dump_path(self) -> None: + # if we want to update Workspace Manifest with data from RAW dump + if ( + self.args.raw_dump_path + and self.args.do_update is True # noqa: W503 + and self.args.just_preview is False # noqa: W503 + ): + dump_path = self.args.raw_dump_path + if self.args.raw_dump_path.is_absolute() is False: + dump_path = os.getcwd() / self.args.raw_dump_path + if self.args.workspace_path: + if self.args.workspace_path.is_absolute() is False: + root_path = os.getcwd() / self.args.workspace_path + else: + root_path = self.args.workspace_path + else: + root_path = os.getcwd() + + if os.path.normpath(dump_path) != os.path.normpath(root_path): + if self.args.use_force is False: + raise Exception( + "Please consider again what you are trying to do.\nYou want to update Manifest in the Workspace by RAW dump, yet you want to start dump not from Workspace root.\nThis may lead to strange Manifest.\nIf you are still sure that this is what you want, use '--force'." # noqa: E501 + ) + + def _allow_only_1_update_on_a_time(self) -> None: + # just 1 update type at a time + if self.args.do_update is True and self.args.update_on: # noqa: W503 + raise Exception("Use only one out of '--update' or '--update-on' at a time") + + def _update_on_file_must_exist(self) -> None: + if self.args.update_on: + # check if provided file actually exists + if self.args.update_on.is_file() is False: + raise Exception("'UPDATE_AT' file does not exists") + + """ + ===================== + obtaining data - part + ===================== + """ + + def _get_dm_load_path(self) -> None: + # obtains load_path as path of Deep Manifest repository + dm_is_dirty: bool = False + + gtf = GroupsToFind(self.args.groups) + dm = None + if self.dmod.workspace: + dm, _ = get_deep_manifest_from_local_manifest_pcsrepo( + self.dmod.workspace, + gtf, + ) + if dm: + self.dmod.update_source_path = ( + self.dmod.workspace.root_path / dm.dest / "manifest.yml" + ) + # look for git status if it is not dirty + gits = GitStatus(self.dmod.workspace.root_path / dm.dest) + gits.update() + dm_is_dirty = gits.dirty + if dm_is_dirty is True: + # verify if 'manifest.yml' alone is dirty + _, out_stat = run_git_captured( + self.dmod.workspace.root_path / dm.dest, + "status", + "--porcelain=1", + "manifest.yml", + check=False, + ) + if out_stat == "": + # cancel dirty flag if 'manifest.yml' is clean + dm_is_dirty = False + + if ( + dm_is_dirty is True + and self.args.use_force is False # noqa: W503 + and not self.args.save_to # noqa: W503 + and self.args.just_preview is False # noqa: W503 + ): + raise Exception( + "not updating Deep Manifest as it is dirty, use '--force' to overide or '--save-to' somewhere else" # noqa: E501 + ) + + if self.dmod.update_source_path: + ui.info_2("Loading Deep Manifest from", self.dmod.update_source_path) + + # save such path as possible output path + self.dmod.final_output_path_list.update_on_path = ( + self.dmod.update_source_path + ) + self.dmod.update_source = UpdateSourceEnum.DEEP_MANIFEST + self.dmod.final_output_mode.append(FinalOutputModeFlag.UPDATE) + + else: + raise Exception("Cannot obtain Deep Manifest from Workspace to update") diff --git a/tsrc/dump_manifest_helper.py b/tsrc/dump_manifest_helper.py new file mode 100644 index 00000000..54551169 --- /dev/null +++ b/tsrc/dump_manifest_helper.py @@ -0,0 +1,100 @@ +""" +Manifest Dumper - Helpers + +helps Dumper to use unified dataclass +that can be processed across various of cases +and thus simplify already complex functions +""" + +from dataclasses import dataclass +from typing import Dict, List, Optional, Union + +import cli_ui as ui + +from tsrc.repo import Remote, Repo +from tsrc.status_endpoint import CollectedStatuses, Status + + +@dataclass(frozen=True) +class ManifestRepoItem: + branch: Optional[str] = None + tag: Optional[str] = None + sha1: Optional[str] = None + empty: Optional[bool] = False + ignore_submodules: Optional[bool] = False + remotes: Optional[List[Remote]] = None + groups_considered: Optional[bool] = False + # TODO: implement test if required variables are set + + @property + def clone_url(self) -> str: + assert self.remotes + return self.remotes[0].url + + +class MRISHelpers: + def __init__( + self, + statuses: Optional[CollectedStatuses] = None, + w_repos: Optional[List[Repo]] = None, # Workspace's Repos + repos: Optional[List[Repo]] = None, + ) -> None: + self.mris: Dict[str, ManifestRepoItem] = {} + if bool(statuses) == bool(repos): + return + if statuses: + self._statuses_to_mris(statuses, w_repos) + if repos: + self._repos_to_mris(repos) + + def _repo_to_mri( + self, + repo: Repo, + ) -> ManifestRepoItem: + return ManifestRepoItem( + branch=repo.branch, + tag=repo.tag, + sha1=repo.sha1, + ignore_submodules=repo.ignore_submodules, + remotes=repo.remotes, + ) + + def _repos_to_mris( + self, + repos: Union[List[Repo], None], + ) -> None: + if repos: + for repo in repos: + # skip empty Repo(s) + if repo.branch or repo.tag or repo.sha1: + self.mris[repo.dest] = self._repo_to_mri(repo) + else: + ui.warning(f"Skipping empty Repo: {repo.dest}") + + def _status_to_mri( + self, + status: Union[Status, Exception], + w_repo: Repo, + ) -> ManifestRepoItem: + if isinstance(status, Status) and status.git.empty is False: + return ManifestRepoItem( + branch=status.git.branch, + tag=status.git.tag, + sha1=status.git.sha1, + empty=status.git.empty, + ignore_submodules=w_repo.ignore_submodules, + remotes=w_repo.remotes, + groups_considered=True, + ) + return ManifestRepoItem() + + def _statuses_to_mris( + self, + statuses: Union[CollectedStatuses, None], + w_repos: Union[List[Repo], None], + ) -> None: + if statuses and w_repos: + for repo in w_repos: + dest = repo.dest + if isinstance(statuses[dest], Status): + self.mris[dest] = self._status_to_mri(statuses[dest], repo) diff --git a/tsrc/dump_manifest_raw_grabber.py b/tsrc/dump_manifest_raw_grabber.py new file mode 100644 index 00000000..cae90826 --- /dev/null +++ b/tsrc/dump_manifest_raw_grabber.py @@ -0,0 +1,172 @@ +import os +from copy import deepcopy +from pathlib import Path +from typing import List, Tuple, Union + +import cli_ui as ui + +from tsrc.dump_manifest_args import DumpManifestArgs +from tsrc.dump_manifest_args_data import DumpManifestOperationDetails +from tsrc.executor import process_items +from tsrc.repo import Repo +from tsrc.repo_grabber import RepoGrabber +from tsrc.utils import erase_last_line + + +class ManifestRawGrabber: + # using paralelism; how it is done: + # 1st: obtain all '.git' paths and just save it to List + # 2nd: call 'process_items' to get GIT stats + def __init__(self, a: DumpManifestArgs, dfp: Path) -> None: + self.a = a + self.dump_from_path = dfp + if self.dump_from_path.is_dir() is False: + raise Exception(f"Such Path is not found: {self.dump_from_path}") + ui.info_1( + *[ + "Checking Path (recursively) for Repos from:", + ui.blue, + self.dump_from_path, + ] + ) + + def _may_reduce_path( + self, int_path: Union[List[str], None], this_path: List[str] + ) -> List[str]: + # remove '.git' from the end and also last dir if exists + len_this_path = len(this_path) + if len_this_path >= 2: + del this_path[-2:] # keep also name of the dir wehre is '.git' + elif len_this_path == 1: + del this_path[-1] + + # find maximum common Path + if int_path: + tmp_list = deepcopy(this_path) + for i, _ in enumerate(tmp_list): + if len(int_path) > i and tmp_list[i] != int_path[i]: + del int_path[i] + else: + int_path = deepcopy(this_path) + return int_path + + def get_common_path(self, use_path: Path) -> Union[List[str], None]: + # return maximum common Path + + int_path: Union[List, None] = None # intersectioned path + some_int_path: bool = False # should we return something? + # for root, dirs, files in os.walk(this_path): + for root, _, _ in os.walk(use_path): + path = root.split(os.sep) + name = os.path.basename(root) + + do_continue: bool = False + # do not consider dot-started dirs, but '.git' + for i, p in enumerate(path): + if i > 0 and p != ".git" and p.startswith(".") is True: + do_continue = True + break + if do_continue is True: + continue + + # we may have found Repo + if name == ".git": + this_path = deepcopy(path) + if len(path) >= 2: + int_path = self._may_reduce_path(int_path, this_path) + some_int_path = True + + if not int_path and some_int_path is True: + return ["."] # try current directory when empty + + return int_path + + def _grab_on_repo_path( + self, path: List[str], common_path: Union[List[str], None] + ) -> Tuple[Union[Path, None], Union[str, None]]: + + use_path: Union[Path, None] = None + use_path_list: Union[List[str], None] = None + this_path = deepcopy(path) + if len(path) >= 2: + del this_path[-1] + use_path = Path(os.sep.join(this_path)) + use_path_list = this_path + + if use_path and use_path_list and common_path: + use_path_clean = deepcopy(use_path_list) + for index, _ in reversed(list(enumerate(use_path_list))): + if ( + len(common_path) > index + and use_path_list[index] == common_path[index] # noqa: W503 + ): + del use_path_clean[index] + + if use_path_clean: + return Path(use_path), os.sep.join(use_path_clean) + + return None, None + + def common_path_is_ready( + self, dump_from_path: Path + ) -> Tuple[Union[List[str], None], DumpManifestOperationDetails]: + # grab_save_path: Union[Path, None] = None + # common_path had to be removed from every Repo find later + common_path = self.get_common_path(dump_from_path) + if common_path: + common_path_path = os.sep.join(common_path) + ui.info_2(f"Using Repo(s) COMMON PATH on: '{common_path_path}'") + + # call args handling (include data and check yet again) + self.a.dmod = self.a.consider_common_path(common_path) + + return common_path, self.a.dmod + + def grab(self, num_jobs: int) -> Tuple[List[Repo], DumpManifestArgs]: + + # let us understand the situation we are in + ui.info_1("Note: it is not possible to obtain anything regarding Groups") + + # verify and fetch 'common_path' ('grab_save_path' optionaly too) + common_path, self.a.dmod = self.common_path_is_ready(self.dump_from_path) + + repos_paths: List[Repo] = [] # here 'dest' is used as Path + + for root, _, _ in os.walk(self.dump_from_path): + path = root.split(os.sep) + name = os.path.basename(root) + + do_continue: bool = False + for i, p in enumerate(path): + if i > 0 and p != ".git" and p.startswith(".") is True: + do_continue = True + break + + if do_continue is True: + continue + if name == ".git": + repo_path, clean_dest = self._grab_on_repo_path(path, common_path) + if not repo_path: + continue + + # create pseudo-Repo for 'process_items' to eat + if repo_path and clean_dest: + this_repo = Repo( + dest=clean_dest, remotes=[], _grabbed_from_path=repo_path + ) + repos_paths.append(this_repo) + + if repos_paths: + # we have now list of Paths of possible Repos + repo_grabber = RepoGrabber(common_path) + ui.info_1(f"Checking for Repos: out of possible {len(repos_paths)} paths") + process_items(repos_paths, repo_grabber, num_jobs=num_jobs) + erase_last_line() + ui.info_2( + f"Found {len(repo_grabber.repos)} Repos out of {len(repos_paths)} possible paths" + ) + + return repo_grabber.repos, self.a + else: + ui.info_2("No Repos were found") + return [], self.a diff --git a/tsrc/errors.py b/tsrc/errors.py index 595b9cd2..82790cc0 100644 --- a/tsrc/errors.py +++ b/tsrc/errors.py @@ -43,8 +43,10 @@ class LoadManifestSchemaError(Error): def __init__(self, mtod: ManifestsTypeOfData) -> None: if mtod == ManifestsTypeOfData.DEEP: msg = "Failed to get Deep Manifest" - if mtod == ManifestsTypeOfData.FUTURE: + elif mtod == ManifestsTypeOfData.FUTURE: msg = "Failed to get Future Manifest" + else: + msg = "Failed to get Manifest" super().__init__(msg) diff --git a/tsrc/file_system.py b/tsrc/file_system.py index 3b6d36dd..2637f473 100644 --- a/tsrc/file_system.py +++ b/tsrc/file_system.py @@ -106,3 +106,13 @@ def check_link(*, source: Path, target: Path) -> bool: if remove_link: os.unlink(source) return True + + +def make_relative(in_path: Path) -> Path: + # takes input Path and make it relative to current path + # if it is not relative already + if in_path.is_absolute() is True: + in_path_dir = os.path.dirname(in_path) + in_path_file = os.path.basename(in_path) + in_path = Path(os.path.join(os.path.relpath(in_path_dir), in_path_file)) + return in_path diff --git a/tsrc/git.py b/tsrc/git.py index ccbd1cc5..690c3c72 100644 --- a/tsrc/git.py +++ b/tsrc/git.py @@ -76,7 +76,6 @@ def __init__(self, working_path: Path) -> None: self.tag: Optional[str] = None self.branch: Optional[str] = None self.sha1: Optional[str] = None - self.upstreamed = False def update(self) -> None: # Try and gather as many information about the git repository as @@ -90,7 +89,6 @@ def update(self) -> None: self.update_tag() self.update_remote_status() self.update_worktree_status() - self.update_upstreamed() def update_sha1(self) -> None: self.sha1 = get_sha1(self.working_path, short=True) @@ -137,27 +135,6 @@ def update_worktree_status(self) -> None: self.added += 1 self.dirty = True - def update_upstreamed(self) -> None: - use_branch = self.branch - # if there is tag with same name as branch, it gets refered by 'heads/' - if use_branch: - if use_branch.startswith("heads/") is True: - use_branch = use_branch[6:] - else: - self.upstreamed = False - # skip git check if upstreamed when there is no branch - return - - rc, _ = run_git_captured( - self.working_path, - "config", - "--get", - f"branch.{use_branch}.remote", - check=False, - ) - if rc == 0: - self.upstreamed = True - def describe(self) -> List[ui.Token]: """Return a list of tokens suitable for ui.info.""" res: List[ui.Token] = [] diff --git a/tsrc/git_remote.py b/tsrc/git_remote.py index becd4d6e..51545320 100644 --- a/tsrc/git_remote.py +++ b/tsrc/git_remote.py @@ -15,6 +15,52 @@ from urllib.parse import quote, urlparse from tsrc.git import run_git_captured +from tsrc.repo import Remote + + +class GitRemote: + def __init__(self, working_path: Path, cur_branch: Union[str, None]) -> None: + self.working_path = working_path + self.branch = cur_branch + self.remotes: List[Remote] = [] + self.upstreamed = False # only refers to current branch + + def update(self) -> None: + self.update_remotes() + if self.remotes and self.branch: + self.update_upstreamed() + + def update_remotes(self) -> None: + # obtain information about configured 'remotes' + # in 'GitStatus' obtaining such information + # is not useful as remotes are stored in Manifest + _, out = run_git_captured(self.working_path, "remote") + for line in out.splitlines(): + _, url = run_git_captured(self.working_path, "remote", "get-url", line) + if line and url: + tmp_r = Remote(name=line, url=url) + self.remotes.append(tmp_r) + + def update_upstreamed(self) -> None: + use_branch = self.branch + # if there is tag with same name as branch, it gets refered by 'heads/' + if use_branch: + if use_branch.startswith("heads/") is True: + use_branch = use_branch[6:] + else: + self.upstreamed = False + # skip git check if upstreamed when there is no branch + return + + rc, _ = run_git_captured( + self.working_path, + "config", + "--get", + f"branch.{use_branch}.remote", + check=False, + ) + if rc == 0: + self.upstreamed = True def remote_urls_are_same(url_1: str, url_2: str) -> bool: @@ -79,6 +125,12 @@ def remote_branch_exist(url: str, branch: str) -> int: return rc +def get_git_remotes(working_path: Path, cur_branch: str) -> GitRemote: + remotes = GitRemote(working_path, cur_branch) + remotes.update() + return remotes + + def get_l_and_r_sha1_of_branch( w_r_path: Path, dest: str, diff --git a/tsrc/groups.py b/tsrc/groups.py index 1408a515..0825691e 100644 --- a/tsrc/groups.py +++ b/tsrc/groups.py @@ -62,6 +62,7 @@ def __init__(self, *, elements: List[T]) -> None: self.groups: Dict[str, Group[T]] = {} self.all_elements = elements self._groups_seen: List[str] = [] + self.missing_elements: List[Dict[str, T]] = [] def get_groups_seen(self) -> List[str]: return self._groups_seen @@ -74,17 +75,23 @@ def add( ignore_on_mtod: Optional[ManifestsTypeOfData] = None, ) -> None: can_add: bool = True + ignored_elements: List[T] = [] for element in elements: if element not in self.all_elements: if ignore_on_mtod: can_add = False - ui.warning( - f"{get_mtod_str(ignore_on_mtod)}: Groups: cannot add '{element}' to '{name}'." # noqa: E501 - ) + if ignore_on_mtod != ManifestsTypeOfData.DEEP_ON_UPDATE: + ui.warning( + f"{get_mtod_str(ignore_on_mtod)}: Groups: cannot add '{element}' to '{name}'." # noqa: E501 + ) + # store missing element, also keep its asignment to the Group name + self.missing_elements.append({name: element}) + ignored_elements.append(element) else: raise UnknownGroupElement(name, element) - if can_add is True: - self.groups[name] = Group(name, elements, includes=includes) + if can_add is False: + elements = list(set(elements).difference(ignored_elements)) + self.groups[name] = Group(name, elements, includes=includes) def get_group(self, name: str) -> Optional[Group[T]]: return self.groups.get(name) diff --git a/tsrc/groups_and_constraints_data.py b/tsrc/groups_and_constraints_data.py new file mode 100644 index 00000000..87ddf415 --- /dev/null +++ b/tsrc/groups_and_constraints_data.py @@ -0,0 +1,36 @@ +""" +Dataclass for Groups, 'include_regex', 'exclude_regex' +'singular_remote' + +everything that can reduce Repos should have single +place and that place should be here +""" + +import argparse +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass(frozen=True) +class GroupsAndConstraints: + groups: Optional[List[str]] = None # just what was provided via cmd + # not to be mistaken with Group class + singular_remote: str = "" + include_regex: str = "" + exclude_regex: str = "" + + +def get_group_and_constraints_data(args: argparse.Namespace) -> GroupsAndConstraints: + + groups: Optional[List[str]] = None + if args.groups: + groups = args.groups + include_regex: str = "" + if args.include_regex: + include_regex = args.include_regex + exclude_regex: str = "" + if args.exclude_regex: + exclude_regex = args.exclude_regex + return GroupsAndConstraints( + groups=groups, include_regex=include_regex, exclude_regex=exclude_regex + ) diff --git a/tsrc/manifest_common_data.py b/tsrc/manifest_common_data.py index 48737f07..f8fa86de 100644 --- a/tsrc/manifest_common_data.py +++ b/tsrc/manifest_common_data.py @@ -11,6 +11,7 @@ class ManifestsTypeOfData(Enum): DEEP_ON_UPDATE = 3 # do not put warning about missing element DEEP_BLOCK = 4 FUTURE = 5 + SAVED = 6 # manifest created by '--save-to' def get_mtod_str(tod: ManifestsTypeOfData) -> str: @@ -24,30 +25,38 @@ def get_mtod_str(tod: ManifestsTypeOfData) -> str: return "Deep Manifest's block" if tod == ManifestsTypeOfData.FUTURE: return "Future Manifest" + if tod == ManifestsTypeOfData.SAVED: + return "Saved Manifest" def mtod_can_ignore_remotes() -> List[ManifestsTypeOfData]: rl: List[ManifestsTypeOfData] = [ + # only for LOCAL Manifest the missing remote + # cannot be ignored. ManifestsTypeOfData.DEEP, ManifestsTypeOfData.DEEP_ON_UPDATE, ManifestsTypeOfData.DEEP_BLOCK, ManifestsTypeOfData.FUTURE, + ManifestsTypeOfData.SAVED, ] return rl -def get_main_color(tod: ManifestsTypeOfData) -> ui.Token: - # TODO: rename with prefix 'mtod' +def mtod_get_main_color(tod: ManifestsTypeOfData) -> ui.Token: # for Local Manifest (using for Manifest's Marker color) if tod == ManifestsTypeOfData.LOCAL: return ui.reset + # for Deep Manifest (for: 'dest' color, MM color) if tod == ManifestsTypeOfData.DEEP: return ui.purple + # for Deep Manifest block (for: square brackets color) if tod == ManifestsTypeOfData.DEEP_BLOCK: return ui.brown + # for Future Manifest (for 'dest' color, MM color) if tod == ManifestsTypeOfData.FUTURE: return ui.cyan - return ui.reset + + return ui.reset # we should never reach it diff --git a/tsrc/repo.py b/tsrc/repo.py index f4c4dbb2..0966916e 100644 --- a/tsrc/repo.py +++ b/tsrc/repo.py @@ -2,11 +2,12 @@ from dataclasses import dataclass from enum import Enum, unique +from pathlib import Path from typing import List, Optional, Tuple import cli_ui as ui -from tsrc.manifest_common_data import ManifestsTypeOfData, get_main_color +from tsrc.manifest_common_data import ManifestsTypeOfData, mtod_get_main_color @unique @@ -47,12 +48,17 @@ class Repo: tag: Optional[str] = None shallow: bool = False ignore_submodules: bool = False + # only used by RepoGrabber + _grabbed_from_path: Optional[Path] = None def __post_init__(self) -> None: if not self.branch and self.keep_branch is False: object.__setattr__(self, "branch", "master") object.__setattr__(self, "is_default_branch", True) + def rename_dest(self, new_dest: str) -> None: + object.__setattr__(self, "dest", new_dest) + @property def clone_url(self) -> str: assert self.remotes @@ -82,7 +88,6 @@ def describe_to_tokens( if self.tag: present_dtt.append(DescribeToTokens.TAG) if not self.remotes: - # TODO: possibly consider FM as well if mtod == ManifestsTypeOfData.DEEP or mtod == ManifestsTypeOfData.FUTURE: present_dtt.append(DescribeToTokens.MISSING_REMOTES) if not present_dtt: @@ -102,7 +107,7 @@ def _describe_to_token_output( cs = ui.red # color (for) SHA1 ct = ui.brown # color (for) tag if mtod == ManifestsTypeOfData.DEEP or mtod == ManifestsTypeOfData.FUTURE: - cb = cs = get_main_color(mtod) + cb = cs = mtod_get_main_color(mtod) res: List[ui.Token] = [] able: List[ui.Token] = [] diff --git a/tsrc/repo_grabber.py b/tsrc/repo_grabber.py new file mode 100644 index 00000000..e2d4fcc3 --- /dev/null +++ b/tsrc/repo_grabber.py @@ -0,0 +1,71 @@ +""" +Repo Grabber + +allows 'dump-manifest' to paralelise GIT operations +on single Path of possible Repo. +""" + +from pathlib import Path +from typing import List, Optional, Union + +import cli_ui as ui + +from tsrc.executor import Outcome, Task +from tsrc.git import GitStatus, is_git_repository +from tsrc.git_remote import GitRemote +from tsrc.repo import Repo + + +class RepoGrabber(Task[Repo]): + """ + Implements a Task that check and obtain Repo from Path + """ + + def __init__(self, common_path: Union[List[str], None]) -> None: + self.common_path = common_path + self.repos: List[Repo] = [] # these are our output data + + def describe_item(self, item: Repo) -> str: + return item.dest + + def describe_process_start(self, item: Repo) -> List[ui.Token]: + return [item.dest] + + def describe_process_end(self, item: Repo) -> List[ui.Token]: + return [ui.green, "ok", ui.reset, item.dest] + + def process(self, index: int, count: int, repo: Repo) -> Outcome: + + # we need actual Path (as Workspace Path may not be present here) + repo_path: Optional[Path] = repo._grabbed_from_path + if repo_path: + if is_git_repository(repo_path) is False: + return Outcome.empty() + + # obtain local GIT data + gits = GitStatus(repo_path) + gits.update() + + # obtain remote GIT data as well + gitr = GitRemote(repo_path, repo.branch) + gitr.update() + if not gitr.remotes: + # report missing remotes as such manifest will have litle meaning + # in case we will want to use it later for synchronization + ui.warning(f"No remote found for: '{repo.dest}' (path: '{repo_path}')") + + # we are now ready to create full Repo + self.repos.append( + Repo( + dest=repo.dest, + branch=gits.branch, + keep_branch=True, # save empty branch if it is empty + is_default_branch=False, + orig_branch=gits.branch, + sha1=gits.sha1, + tag=gits.tag, + remotes=gitr.remotes, + ) + ) + + return Outcome.empty() diff --git a/tsrc/status_endpoint.py b/tsrc/status_endpoint.py index 1942ad41..d1df17ad 100644 --- a/tsrc/status_endpoint.py +++ b/tsrc/status_endpoint.py @@ -6,7 +6,9 @@ from tsrc.errors import MissingRepoError from tsrc.executor import Outcome, Task from tsrc.git import GitStatus, get_git_status +from tsrc.git_remote import GitRemote, get_git_remotes from tsrc.manifest import Manifest +from tsrc.manifest_common_data import ManifestsTypeOfData from tsrc.repo import Repo from tsrc.utils import erase_last_line from tsrc.workspace import Workspace @@ -20,8 +22,9 @@ def __init__(self, repo: Repo, *, manifest: Manifest): self.manifest = manifest self.incorrect_branch: Optional[Tuple[str, str]] = None self.missing_upstream = True + self.git_remote: Union[GitRemote, None] = None - def update(self, git_status: GitStatus) -> None: + def update(self, git_status: GitStatus, git_remote: Union[GitRemote, None]) -> None: """Set self.incorrect_branch if the local git status does not match the branch set in the manifest. """ @@ -29,7 +32,9 @@ def update(self, git_status: GitStatus) -> None: actual_branch = git_status.branch if actual_branch and expected_branch and actual_branch != expected_branch: self.incorrect_branch = (actual_branch, expected_branch) - self.missing_upstream = not git_status.upstreamed + if git_remote: + self.missing_upstream = not git_remote.upstreamed + self.git_remote = git_remote def describe(self) -> List[ui.Token]: """Return a list of tokens suitable for ui.info()`.""" @@ -38,7 +43,9 @@ def describe(self) -> List[ui.Token]: if incorrect_branch: actual, expected = incorrect_branch res += [ui.red, "(expected: " + expected + ")"] - if self.missing_upstream: + if self.git_remote and not self.git_remote.remotes: + res += [ui.red, "(missing remote)"] + elif self.missing_upstream: res += [ui.red, "(missing upstream)"] return res @@ -46,8 +53,15 @@ def describe(self) -> List[ui.Token]: class Status: """Wrapper class for both ManifestStatus and GitStatus""" - def __init__(self, *, git: GitStatus, manifest: ManifestStatus): + def __init__( + self, + *, + git: GitStatus, + git_remote: Union[GitRemote, None], + manifest: ManifestStatus + ): self.git = git + self.git_remote = git_remote self.manifest = manifest @@ -56,9 +70,12 @@ class StatusCollector(Task[Repo]): stats w.r.t the manifest for each repo. """ - def __init__(self, workspace: Workspace) -> None: + def __init__(self, workspace: Workspace, ignore_group_item: bool = False) -> None: self.workspace = workspace - self.manifest = workspace.get_manifest() + if ignore_group_item is True: + self.manifest = workspace.get_manifest_safe_mode(ManifestsTypeOfData.LOCAL) + else: + self.manifest = workspace.get_manifest() self.statuses: CollectedStatuses = collections.OrderedDict() def describe_item(self, item: Repo) -> str: @@ -80,9 +97,60 @@ def process(self, index: int, count: int, repo: Repo) -> Outcome: self.statuses[repo.dest] = MissingRepoError(repo.dest) try: git_status = get_git_status(full_path) + git_remote: Union[GitRemote, None] = None + if git_status.branch: + git_remote = get_git_remotes(full_path, git_status.branch) manifest_status = ManifestStatus(repo, manifest=self.manifest) - manifest_status.update(git_status) - status = Status(git=git_status, manifest=manifest_status) + manifest_status.update(git_status, git_remote) + status = Status( + git=git_status, git_remote=git_remote, manifest=manifest_status + ) + self.statuses[repo.dest] = status + except Exception as e: + self.statuses[repo.dest] = e + if not self.parallel: + erase_last_line() + return Outcome.empty() + + +class StatusCollectorLocalOnly(Task[Repo]): + """Implement a Task to collect local git status and + stats w.r.t the manifest for each repo. + Only considering local properties of Repo. + This is meant to speed up processing as checking all remotes + can be time consuming. + """ + + def __init__(self, workspace: Workspace, ignore_group_item: bool = False) -> None: + self.workspace = workspace + if ignore_group_item is True: + self.manifest = workspace.get_manifest_safe_mode(ManifestsTypeOfData.LOCAL) + else: + self.manifest = workspace.get_manifest() + self.statuses: CollectedStatuses = collections.OrderedDict() + + def describe_item(self, item: Repo) -> str: + return item.dest + + def describe_process_start(self, item: Repo) -> List[ui.Token]: + return [item.dest] + + def describe_process_end(self, item: Repo) -> List[ui.Token]: + return [] + + def process(self, index: int, count: int, repo: Repo) -> Outcome: + # Note: Outcome is always empty here, because we + # use self.statuses in the main `run()` function instead + # of calling OutcomeCollection.print_summary() + full_path = self.workspace.root_path / repo.dest + self.info_count(index, count, repo.dest, end="\r") + if not full_path.exists(): + self.statuses[repo.dest] = MissingRepoError(repo.dest) + try: + git_status = get_git_status(full_path) + manifest_status = ManifestStatus(repo, manifest=self.manifest) + manifest_status.update(git_status, None) + status = Status(git=git_status, git_remote=None, manifest=manifest_status) self.statuses[repo.dest] = status except Exception as e: self.statuses[repo.dest] = e diff --git a/tsrc/status_header.py b/tsrc/status_header.py index cd40fe1d..bea74c48 100644 --- a/tsrc/status_header.py +++ b/tsrc/status_header.py @@ -22,7 +22,7 @@ # import tsrc.config_status_rc from tsrc.config_status_rc import ConfigStatusReturnCode -from tsrc.manifest_common_data import ManifestsTypeOfData, get_main_color +from tsrc.manifest_common_data import ManifestsTypeOfData, mtod_get_main_color from tsrc.status_header_dm import StatusHeaderDisplayMode from tsrc.workspace import Workspace @@ -129,7 +129,10 @@ def _is_plural(self, value: int, ext: str) -> str: def _header_manifest_url(self, url: str) -> None: ui.info_1( - "Manifest's URL:", get_main_color(ManifestsTypeOfData.DEEP), url, ui.reset + "Manifest's URL:", + mtod_get_main_color(ManifestsTypeOfData.DEEP), + url, + ui.reset, ) def _header_manifest_branch( diff --git a/tsrc/test/cli/test_dump_manifest.py b/tsrc/test/cli/test_dump_manifest.py new file mode 100644 index 00000000..834a1dce --- /dev/null +++ b/tsrc/test/cli/test_dump_manifest.py @@ -0,0 +1,1455 @@ +""" +test_dump_manifest + +all tests regarding 'dump_manifest' command + +OVERVIEW of tests (1st: option(s), 2nd: fn_name, 3rd: additional description): + +--raw --workspace --preview + test_raw_dump__workspace__combined_path + how does it end when --workspace and --raw is provided + +--raw --update" + test_raw_on_update__check_for_remotes__warning + test if Warning is printed even ater UPDATE is completed + +--raw --preview + test_raw_dump_preview + testing ‘--preview’ + +--raw --update-on + test_raw_dump_update_on + updating DM file in the Workspace (provided by hand) + +--raw --update + test_raw_dump_update__use_workspace__without_workspace + test if it skip non-existant DM, Warning should be displayed + +--raw --update --force + test_raw_dump_update_with_multi_remotes__by_load_manifest + test: Update on ‘remotes’ + +--raw --update --save-to --no-repo-delete + test_raw_dump_update_with_multi_remotes__save_to__by_load_manifest + test if we can update and save to another file, while keeping original + Manifest intact + +--raw + test_raw_dump_1_repo_no_workspace__deep_path + COMMON PATH calculation test for 1 Repo + +--raw + test_raw_dump_2_repos_no_workspace__deep_path + COMMON PATH calculation test for 2 Repos (there may be difference on calculation) + +--raw + test_raw_dump__point_blank__no_luck + point black range on Repo, should FAIL + +--raw + test_raw_dump_1_repo_no_workspace__long_input_path + long Path on input on '--raw’ + +--raw --save-to + test_raw_dump_save_to + test SAVE_TO option + +=== === === Workspace only + +--update + test_update_update__by_status + test: update update: all: add, del, update + +--update + test_on_update + test actual update on Workspace (using 'status') + +--update + test_on_update__check_for_remotes__after_update_is_ok + test if UPDATE adds remote(s) if there are none in the current Manifest + +--update + test_on_update__check_for_remotes__is_ok + test warning about Manifest not useful (when no remote is found) +""" + +import os +import re +from pathlib import Path +from shutil import move +from typing import List, Optional, Tuple + +# import pytest +import ruamel.yaml +from cli_ui.tests import MessageRecorder + +from tsrc.git import run_git +from tsrc.manifest import Manifest, load_manifest, load_manifest_safe_mode +from tsrc.manifest_common_data import ManifestsTypeOfData +from tsrc.repo import Repo +from tsrc.test.helpers.cli import CLI +from tsrc.test.helpers.git_server import GitServer +from tsrc.test.helpers.message_recorder_ext import MessageRecorderExt +from tsrc.workspace_config import WorkspaceConfig + + +def test_raw_dump_save_to( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test RAW dump and '--save-to' option + + Scenario: + + * 1st: create dir with sub-dir in it + * 2nd: create 'repo 1', GIT init, add, commit + * 3rd: RAW dump with different --save-to path + * 4th: verify last command output + """ + # 1st: create dir with sub-dir in it + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + + # 2nd: create 'repo 1', GIT init, add, commit + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 3rd: RAW dump with different --save-to path + os.chdir(workspace_path) + tsrc_cli.run("dump-manifest", "--raw", "common path lvl1", "--save-to", sub1_path) + + # 4th: verify last command output + assert message_recorder.find( + r":: Checking Path \(recursively\) for Repos from: common path lvl1" + ) + assert message_recorder.find(r"=> Found 1 Repos out of 1 possible paths") + assert message_recorder.find( + r"=> Creating NEW file 'common path lvl1.level 2.manifest.yml'" + ) + assert message_recorder.find(r":: Dump complete") + + +def test_raw_dump_1_repo_no_workspace__long_input_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + similar than 'test_raw_dump_1_repo_no_workspace__deep_path' + + here we provide long path for RAW option to dump from + + Scenario: + + * 1st: create dir with sub-dir in it + * 2nd: create 'repo 1', GIT init, add, commit + * 3rd: call: RAW dump on ==> deep path <== + * 4th: verify that everything goes ok + """ + + # 1st: create dir with sub-dir in it + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + + # 2nd: create 'repo 1', GIT init, add, commit + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 3rd: call: RAW dump on deep path + os.chdir(workspace_path) + message_recorder.reset() + raw_from_str = os.path.join(".", "common path lvl1", "level 2") + tsrc_cli.run("dump-manifest", "--raw", raw_from_str) + + # 4th: verify that everything goes ok + assert message_recorder.find( + r"Warning: No remote found for: 'repo 1' \(path: 'common path lvl1.*level 2.*repo 1'\)" + ) + assert message_recorder.find(r"=> Found 1 Repos out of 1 possible paths") + assert message_recorder.find( + r"=> Creating NEW file 'common path lvl1.*level 2.*manifest.yml'" + ) + assert message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + assert message_recorder.find(r":: Dump complete") + + # 5th: verify by 'load_manifest_safe_mode' + m_file = workspace_path / sub1_path / "manifest.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count: int = 0 + for repo in m.get_repos(): + if repo.dest == "repo 1": + count += 1 + else: + raise Exception("Manifest contain wrong item") + if count != 1: + raise Exception("Manifest does not contain all items") + + +def test_raw_dump_from_abs_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if COMMON PATH and Current PATH is handled properly + Which one will be checked needs to be decided >>BEFORE<< + it gets checked, so we do not end up with unnecessary + Error about file already exists + + Scenario: + + * 1st: create deep path + * 2nd: create some repos there + * 3rd: add Manifest repository there + * 4th: init workspace (in sub-dir of 2nd level) on master + * 5th: create different sub-dir (from original root dir) + * 6th: create 'repo 3', GIT init, add, commit + * 7th: creates Manifest dump from Workspace to default location + * 8th: creates RAW Manifest dump to COMMON PATH + * 9th: now when trying to dump Manifest from Workspace it should fail + * 10th: move already created Manifest to free default file-name + * 11th: now it should create successfully + * 12th: now RAW dump should fail as file in COMMON PATH already exists + * 13th: if we move "repo 3", than COMMON PATH calculation will + """ + # 1st: create deep path + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + os.chdir(sub1_path) + + # 2nd: create some repos there + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 3rd: add Manifest repository there + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 4th: init workspace (in sub-dir of 2nd level) on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / sub1_path / ".tsrc" / "config.yml") + + # 5th: create different sub-dir (from original root dir) + os.chdir(workspace_path) + sub2_path = os.path.join("common path lvl1") + os.chdir(sub2_path) + + # 6th: create 'repo 3', GIT init, add, commit + sub2_1_path = Path("repo 3") + os.mkdir(sub2_1_path) + os.chdir(sub2_1_path) + full2_path: Path = Path(os.path.join(workspace_path, sub2_path, sub2_1_path)) + run_git(full2_path, "init") + sub2_1_1_file = Path("in_repo.txt") + sub2_1_1_file.touch() + run_git(full2_path, "add", "in_repo.txt") + run_git(full2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 7th: creates Manifest dump from Workspace to default location + os.chdir(workspace_path) + tsrc_cli.run("dump-manifest", "--workspace", sub1_path) + + # 8th: creates RAW Manifest dump to COMMON PATH + # using absolute path for a change + message_recorder.reset() + dump_raw_path = os.path.join(workspace_path, sub2_path) + tsrc_cli.run("dump-manifest", "--raw", dump_raw_path) + assert message_recorder.find( + r"=> Creating NEW file '.*common path lvl1.*manifest\.yml'" + ) + + # 9th: now when trying to dump Manifest from Workspace it should fail + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--workspace", sub1_path) + assert message_recorder.find( + r"Error: such file 'manifest.yml' already exists, use '--force' to overwrite it" + ) + + # 10th: move already created Manifest to free default file-name + os.rename("manifest.yml", "old-manifest.yml") + + # 11th: now it should create successfully + # this checks if previous RAW dump does not interfere + # with Workspace dump + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--workspace", sub1_path) + assert message_recorder.find(r"=> Creating NEW file 'manifest.yml'") + + # 12th: now RAW dump should fail as file in COMMON PATH already exists + tsrc_cli.run("dump-manifest", "--raw", ".") + assert message_recorder.find( + r"Error: Such file '.*common path lvl1.*manifest\.yml' already exists, use '--force' if you want to overwrite it" + ) + + # 13th: if we move "repo 3", than COMMON PATH calculation will + # differ. and that allows saving Manifest to different location + # where there is no Manifest with default name + move(str(workspace_path / sub2_path / sub2_1_path), str(workspace_path / sub1_path)) + tsrc_cli.run("dump-manifest", "--raw", ".") + assert message_recorder.find( + # r"=> Creating NEW file '.*common path lvl1.*level 2.*manifest\.yml'" + r"=> Creating NEW file '.*manifest\.yml'" + ) + + +def test_raw_dump__point_blank__no_luck( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test if it is possible to RAW dump Manifest + from point blank range, meaning from exact place + where Repo is. + + Something like this is not possible yet. + In theory we can take previous directory name + and use it as 'dest' and save manifest also there. + + But that is bad choice. If you want, you can + change directory to previous one and try dump-manifest + from there. + + Scenario: + + * 1st: init GIT repository in the root of given directory + * 2nd: add and commit single file there + * 3rd: let us see what 'dump-manifest' in RAW grab of current directory will say + """ + # 1st: init GIT repository in the root of given directory + run_git(workspace_path, "init") + + # 2nd: add and commit single file there + thisfile = Path("this_file.txt") + thisfile.touch() + run_git(workspace_path, "add", "this_file.txt") + run_git(workspace_path, "commit", "-m", "adding this_file.txt") + + # 3rd: let us see what 'dump-manifest' in RAW grab of current directory will say + # with how it is now working, it should fail + # and report no Repos found + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".") + assert message_recorder.find(r"=> No Repos were found") + assert message_recorder.find(r"Error: cannot obtain data: no Repos were found") + + +def test_raw_dump__workspace__combined_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test if '--workspace ' is respected when + RAW dump path is relatvie (not absolute) + + Scenario: + + * 1st: create dir with sub-dir in it + * 2nd: create 'repo 1', GIT init, add, commit + * 3rd: go back to original Path, Dump from there + * 4th: test if '--workspace' gets combined with RAW dump path + """ + + # 1st: create dir with sub-dir in it + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + + # 2nd: create 'repo 1', GIT init, add, commit + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 3rd: go back to original Path, Dump from there + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run( + "dump-manifest", + "--raw", + "level 2", + "--workspace", + "common path lvl1", + "--preview", + ) + + # 4th: test if '--workspace' gets combined with RAW dump path + assert message_recorder.find( + r":: Checking Path \(recursively\) for Repos from: .*common path lvl1.*level 2" + ) + assert message_recorder.find( + r"=> Using Repo\(s\) COMMON PATH on: '.*common path lvl1.*level 2'" + ) + test_path_1: str = os.path.join(sub1_path, "repo 1") + assert message_recorder.find( + r"Warning: No remote found for: 'repo 1' \(path: '.*common path lvl1.*level 2.*repo 1.*'\)" + ) + assert message_recorder.find(r"dest: repo 1") + + +def test_raw_dump_2_repos_no_workspace__deep_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + verify if common_path is properly calculated + when Repos are on deep path + + Scenario: + + * 1st: create dir with sub-dir in it + * 2nd: create 'repo 1', GIT init, add, commit + * 3rd: create 'repo 2', GIT init, add, commit + * 4th: in previous root, call: RAW dump on '.' + * 5th: check the further output of commnad + * 6th: verify by 'load_manifest_safe_mode' + """ + + # 1st: create dir with sub-dir in it + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + + # 2nd: create 'repo 1', GIT init, add, commit + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 3rd: create 'repo 2', GIT init, add, commit + os.chdir(workspace_path) + os.chdir(sub1_path) + sub1_2_path = Path("repo 2") + os.mkdir(sub1_2_path) + os.chdir(sub1_2_path) + full2_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_2_path)) + run_git(full2_path, "init") + sub1_2_1_file = Path("in_repo.txt") + sub1_2_1_file.touch() + run_git(full2_path, "add", "in_repo.txt") + run_git(full2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 4th: in previous root, call: RAW dump on '.' + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".") + + # 5th: check the further output of commnad + # includes checking COMMON PATH + # and Warning about missing remotes + assert message_recorder.find( + r"=> Using Repo\(s\) COMMON PATH on: '\..*common path lvl1.*level 2'" + ) + + assert message_recorder.find( + r"Warning: No remote found for: 'repo 1' \(path: 'common path lvl1.*level 2.*repo 1'\)" + ) + assert message_recorder.find( + r"Warning: No remote found for: 'repo 2' \(path: 'common path lvl1.*level 2.*repo 2'\)" + ) + assert message_recorder.find(r"=> Found 2 Repos out of 2 possible paths") + assert message_recorder.find( + r"=> Creating NEW file 'common path lvl1.*level 2.*manifest.yml'" + ) + assert message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + assert message_recorder.find(r":: Dump complete") + + # 6th: verify by 'load_manifest_safe_mode' + m_file = workspace_path / sub1_path / "manifest.yml" + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count: int = 0 + for repo in m.get_repos(): + if repo.dest == "repo 1": + count += 1 + elif repo.dest == "repo 2": + count += 2 + else: + raise Exception("Manifest contain wrong item") + if count != 3: + raise Exception("Manifest does not contain all items") + + +def test_raw_dump_1_repo_no_workspace__deep_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + verify if just 1 Repo can be properly dumped + to Manifest without tsrc Workspace at all + when Repo is on deep path + + test shows that when using just 1 Repo it is different + when using more Repos, thus we also need to check for + just 1 Repo + + Scenario: + + * 1st: create dir with sub-dir in it + * 2nd: create 'repo 1' dir + * 3rd: GIT: init, add, commit + * 4th: in previous root, call: RAW dump on '.' + * 5th: check COMMON PATH and "No remotes" Warning + * 6th: verify by 'load_manifest_safe_mode' + """ + + # 1st: create dir with sub-dir in it + sub1_path = os.path.join("common path lvl1", "level 2") + os.makedirs(sub1_path) + + # 2nd: create 'repo 1' dir + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + + # 3rd: GIT: init, add, commit + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 4th: in previous root, call: RAW dump on '.' + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".") + + # 5th: check COMMON PATH and "No remotes" Warning + assert message_recorder.find( + r"=> Using Repo\(s\) COMMON PATH on: '\..*common path lvl1.*level 2'" + ) + assert message_recorder.find( + r"Warning: No remote found for: 'repo 1' \(path: 'common path lvl1.*level 2.*repo 1'\)" + ) + assert message_recorder.find(r"=> Found 1 Repos out of 1 possible paths") + assert message_recorder.find( + r"=> Creating NEW file 'common path lvl1.*level 2.*manifest.yml'" + ) + assert message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + assert message_recorder.find(r":: Dump complete") + + # 6th: verify by 'load_manifest_safe_mode' + m_file = workspace_path / sub1_path / "manifest.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count: int = 0 + for repo in m.get_repos(): + if repo.dest == "repo 1": + count += 1 + else: + raise Exception("Manifest contain wrong item") + if count != 1: + raise Exception("Manifest does not contain all items") + + +def test_raw_dump_update_with_multi_remotes__by_load_manifest( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, +) -> None: + """ + Test how does 'dump-manifest' is able to update + Manifest repository in the Workspace while using RAW dump. + Create the worst possible conditions to work with. + + Scenario: + + * 1st: Create repositories (1x multiple remotes) + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: remove "origin" remote from 'manifest/manifest.yml" + * 5th: remove "upstream" remote from git of Manifest repository + * 6th: modify Deep Manifest (so update will not be ignored) + * 7th: test actual RAW dump while updating Manifest repo (use force) + * 8th: check by loading updated Manifest + """ + # 1st: Create repositories (1x multiple remotes) + # 'repo1-mr' will have multiple remotes + repo1_url = git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "CMakeLists.txt") + git_server.manifest.set_repo_remotes( + "repo1-mr", [("origin", repo1_url), ("upstream", "git@upstream.com")] + ) + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: remove "origin" remote from 'manifest/manifest.yml" + manifest_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_path) + + # 5th: remove "upstream" remote from git of Manifest repository + repo1_path = workspace_path / "repo1-mr" + run_git(repo1_path, "remote", "remove", "upstream") + + # 6th: modify Deep Manifest (so update will not be ignored) + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_insert_repo_item_to_manifest(manifest_file_path, "does_not_matter") + + # 7th: test actual RAW dump while updating Manifest repo (use force) + tsrc_cli.run("dump-manifest", "--raw", ".", "--update", "--force") + + # 8th: check by loading updated Manifest + """ + here we have Manifest with wrong remote on 'repo1-mr' + so the update should check git and obtain the correct remote (name and url) + and update such data on Manifest. So we should see this state + when we load the Manifest (by 'load_manifest') + """ + w_m_path = workspace_path / "manifest" / "manifest.yml" + m = load_manifest(w_m_path) + for repo in m.get_repos(): + if repo.dest == "repo1-mr": + for remote in repo.remotes: + pattern = re.compile(".*bare.repo1-mr") + if not (remote.name == "origin" and pattern.match(remote.url)): + raise Exception("Wrong repo remotes") + + +def test_raw_dump_update_with_multi_remotes__save_to__by_load_manifest( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test how does 'dump-manifest' is able to update + Manifest repository in the Workspace while using RAW dump. + Create the worst possible conditions to work with. + + Scenario: + + * 1st: Create repositories (1x multiple remotes) + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: remove "origin" remote from 'manifest/manifest.yml" + * 5th: remove "upstream" remote from git of Manifest repository + * 6th: modify Deep Manifest (so update will not be ignored) + * 7th: test actual RAW dump while updating, but saving to another file + * 8th: verify no change to original 'manifest/manifest.yml' + * 9th: verify newly created Manifest: 'new-m.yml' + """ + # 1st: Create repositories (1x multiple remotes) + # 'repo1-mr' will have multiple remotes + repo1_url = git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "CMakeLists.txt") + git_server.manifest.set_repo_remotes( + "repo1-mr", + [ + ("origin", repo1_url), + ("upstream", "git@upstream.com"), + ("debug", "git@debug.com"), + ], + ) + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: remove "origin" remote from 'manifest/manifest.yml" + manifest_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_path) + + # 5th: remove "upstream" remote from git of Manifest repository + repo1_path = workspace_path / "repo1-mr" + run_git(repo1_path, "remote", "remove", "upstream") + + # 6th: modify Deep Manifest (so update will not be ignored) + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_insert_repo_item_to_manifest(manifest_file_path, "does_not_matter") + + # 7th: test actual RAW dump while updating, but saving to another file + # also does not allow Repos to be deleted + # * in this case it should not matter if DM is dirty as we are not overwriting it + # but instead saving output to another file + message_recorder.reset() + tsrc_cli.run( + "dump-manifest", + "--raw", + ".", + "--update", + "--save-to", + "new-m.yml", + "--no-repo-delete", + ) + assert message_recorder.find( + r"=> Creating NEW file 'new-m\.yml' by UPDATING Deep Manifest on 'manifest.manifest\.yml'" + ) + + # 8th: verify no change to original 'manifest/manifest.yml' + on_8th_point(workspace_path) + + # 9th: verify newly created Manifest: 'new-m.yml' + # * verify remotes of 'repo1-mr' + # * verify 'repo5' as it should not be deleted + on_9th_point(workspace_path) + + +def on_8th_point(workspace_path: Path) -> None: + found_remote: bool = False + found_remote_2: bool = False + w_m_path = workspace_path / "manifest" / "manifest.yml" + d_m = load_manifest(w_m_path) + for repo in d_m.get_repos(): + if repo.dest == "repo1-mr": + for remote in repo.remotes: + pattern = re.compile(".*bare.repo1-mr") + if remote.name == "origin" and pattern.match(remote.url): + raise Exception("Wrong remote that should not be here") + if remote.name == "debug" and remote.url == "git@debug.com": + found_remote_2 = True + if remote.name == "upstream" and remote.url == "git@upstream.com": + found_remote = True + if found_remote is False or found_remote_2 is False: + raise Exception("Wrong repo remotes") + + +def on_9th_point(workspace_path: Path) -> None: + n_m_path = workspace_path / "new-m.yml" + m = load_manifest(n_m_path) + found_not_deleted: bool = False + found_remote: bool = False + found_remote_2: bool = False + for repo in m.get_repos(): + if repo.dest == "repo1-mr": + found_remote, found_remote_2 = on_9th_point_on_repo1_mr(repo) + + if repo.dest == "repo5": + found_not_deleted = True + if repo.clone_url != "does_not_matter": + raise Exception("Wrong repo clone_url") + + if found_remote is False or found_remote_2 is False: + raise Exception("Wrong repo remotes") + if found_not_deleted is False: + raise Exception("Missing repo") + + +def on_9th_point_on_repo1_mr(repo: Repo) -> Tuple[bool, bool]: + found_remote: bool = False + found_remote_2: bool = False + for remote in repo.remotes: + pattern = re.compile(".*bare.repo1-mr") + if remote.name == "origin" and pattern.match(remote.url): + found_remote = True + if remote.name == "debug" and remote.url == "git@debug.com": + found_remote_2 = True + if remote.name == "upstream": + raise Exception("Wrong remote that should not be here") + return found_remote, found_remote_2 + + +def test_raw_dump_update__use_workspace__without_workspace( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Check if Workspace gets ignored even if it is called to be used + + Scenario: + + # 1nd: create 'repo 1', GIT init, add, commit + # 2nd: try to dump manifest by RAW mode, while want to update DM + """ + # 1nd: create 'repo 1', GIT init, add, commit + sub1_path = workspace_path + os.chdir(sub1_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init", "-b", "master") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 2nd: try to dump manifest by RAW mode, while want to update DM + # this should give Warning as there is no Workspace, + # nor there is Deep Manifest. in such case, the Workspace + # should be ignored, RAW dump should continue without it + os.chdir(sub1_path) + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + + assert message_recorder.find(r"Error: Could not find current workspace") + + +# flake8: noqa: C901 +def ad_hoc_delete_remote_from_manifest( + manifest_path: Path, +) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + yaml = ruamel.yaml.YAML(typ="rt") + parsed = yaml.load(manifest_path.read_text()) + + for _, value in parsed.items(): + if isinstance(value, List): + for x in value: + if isinstance(x, ruamel.yaml.comments.CommentedMap): + if "dest" in x and x["dest"] == "repo1-mr": + if "remotes" in x: + idx_to_del: Optional[int] = None + remotes = x["remotes"] + for idx, _ in enumerate(remotes): + if remotes[idx]["name"] == "origin": + idx_to_del = idx + break + if isinstance(idx_to_del, int): + del remotes[idx_to_del] + if "url" in x: + del x["url"] + + # write the file down + with open(manifest_path, "w") as file: + yaml.dump(parsed, file) + + +def test_raw_dump_update_on( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + RAW dump test using '--update-on' + (we can see the benefits of RAW dump here) + + Scenario: + + * 1st: Create repositories + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: add some other repos, without addit them to Manifest + * 5th: see that Manifest does not contain latest repos + * 6th: dump Manifest by RAW dump + '--update-on' + * 7th: now test how RAW dump updates Manifest + """ + + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: add some other repos, without addit them to Manifest + just_clone_repo( + git_server, workspace_path, "repo3", branch="old_master", add_to_manifest=False + ) + just_clone_repo( + git_server, workspace_path, "repo4", branch="old_master", add_to_manifest=False + ) + + # 5th: see that Manifest does not contain latest repos + message_recorder.reset() + tsrc_cli.run("status") + assert not message_recorder.find(r"repo3") + assert not message_recorder.find(r"repo4") + + # 6th: dump Manifest by RAW dump + '--update-on' + message_recorder.reset() + # prepare Deep Manfiest the path + manifest_path = os.path.join("manifest", "manifest.yml") + tsrc_cli.run("dump-manifest", "--raw", ".", "--update-on", manifest_path) + # assert message_recorder.find(f"=> Updating on: '{manifest_path}'") + assert message_recorder.find(r"=> UPDATING 'manifest.manifest\.yml'") + + # 7th: now test how RAW dump updates Manifest + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"\* manifest \[ master \]= master \(dirty\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\+ repo4 \[ old_master \] old_master") + assert message_recorder.find(r"\+ repo3 \[ old_master \] old_master") + assert message_recorder.find(r"\* repo2 \[ master \] master") + assert message_recorder.find(r"\* repo1 \[ master \] master") + + +def test_raw_dump_preview( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder_ext: MessageRecorderExt, +) -> None: + """ + Real test of dumping manifest from RAW Repos (no Workspace) + Testing feature: '--preview' + + Scenario: + + * 1st: clone repos without init of Workspace + * 2nd: call RAW dump using preview mode + * 3rd: verify output using 'find_right_after' feature + """ + # 1st: clone repos without init of Workspace + just_clone_repo(git_server, workspace_path, "repo1") + just_clone_repo(git_server, workspace_path, "repo2") + + # 2nd: call RAW dump using preview mode + # in such case Manifest goes to (ui.info) not to file + message_recorder_ext.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--preview") + + # 3rd: verify output using 'find_right_after' feature + assert message_recorder_ext.find(r"^repos:$") + assert message_recorder_ext.find(r"^ - dest: repo1$") + # "url" may be splitted into 2 lines + ret = message_recorder_ext.find_right_after(r"^ url: .*bare.repo1$") + if not ret: + assert message_recorder_ext.find_right_after(r"^ url:") + assert message_recorder_ext.find_right_after(r".*bare.repo1$") + + assert message_recorder_ext.find_right_after(r"^ branch: master$") + assert message_recorder_ext.find(r"^ - dest: repo2$") + ret = message_recorder_ext.find_right_after(r"^ url: .*bare.repo2$") + if not ret: + assert message_recorder_ext.find_right_after(r"^ url:") + assert message_recorder_ext.find_right_after(r".*bare.repo2$") + + assert message_recorder_ext.find_right_after(r"^ branch: master$") + + +def just_clone_repo( + git_server: GitServer, + workspace_path: Path, + dest: str, + branch: str = "master", + add_to_manifest: bool = True, +) -> None: + git_server.add_repo(dest, default_branch=branch, add_to_manifest=add_to_manifest) + git_server.push_file(dest, "CMakeLists.txt", branch=branch) + + repo_url = git_server.get_url(dest) + run_git(workspace_path, "clone", "-b", branch, repo_url) + + +""" +(Below) With Workspace Dump (no RAW dump) +""" + + +def test_update_update__by_status( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test update_update: + this means only Repo's items will need update. + what are these Repo's items? + * branch + * tag + * sha1 + * (remotes is tested in different test) + + NOTE: 'update' will change presence of Repos, + 'update_update' changes Repo's items + + Scenario: + + * 1st: Create repositories + - A) test for 'add' + - B) test for 'del' (and 'add' SHA1) + - C) test for 'update' (updating branch) + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: now let us make some changes on Repo->item level + - A) test for 'add' + - B) test for 'del' (and 'add' SHA1) + - C) test for 'update' (updating branch) + * 5th: Manifest: checkout to 'snapshot' + tag + * 6th: do 'dump-manifest' as update + * 7th: verify 'status' + """ + # 1st: Create repositories + # A) test for 'add' + git_server.add_repo("repo1", default_branch="main") + # git_server.change_branch("repo1", "main") + git_server.push_file("repo1", "CMakeLists.txt", branch="main") + git_server.tag("repo1", "v1.0", branch="main") + + # B) test for 'del' (and 'add' SHA1) + git_server.add_repo("repo2", default_branch="main") + + git_server.push_file("repo2", "test.txt", branch="main") + repo2_sha1_2 = git_server.get_sha1("repo2") + + git_server.push_file("repo2", "test2.txt", branch="main") + repo2_sha1_3 = git_server.get_sha1("repo2") + + git_server.push_file("repo2", "test3.txt", branch="main") + + # C) test for 'update' (updating branch) + git_server.add_repo("repo3", default_branch="main") + git_server.push_file("repo3", "test.txt", branch="main") + + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: now let us make some changes on Repo->item level + # to check how does update_update working + + # A) test for 'add' + repo1_path = workspace_path / "repo1" + run_git(repo1_path, "checkout", "-b", "devel") + + # B) test for 'del' (and 'add' SHA1) + repo2_path = workspace_path / "repo2" + run_git(repo2_path, "checkout", repo2_sha1_2) + run_git(repo2_path, "checkout", "-b", "middle") + run_git(repo2_path, "merge", "main") + run_git(repo2_path, "checkout", repo2_sha1_3) + + # C) test for 'update' (updating branch) + repo3_path = workspace_path / "repo3" + run_git(repo3_path, "checkout", "-b", "point_c") + + # 5th: Manifest: checkout to 'snapshot' + tag + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-b", "snapshot") + run_git(manifest_path, "tag", "-a", "moded", "-m", "all moded version") + run_git(manifest_path, "push", "-u", "origin", "snapshot") + + # 6th: do 'dump-manifest' as update + tsrc_cli.run("dump-manifest", "--update") + + # 7th: verify 'status' + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"\* repo1 \[ devel on v1.0 \] devel on v1.0 \(expected: main\) \(missing upstream\)" # noqa: E501 + ) + # also compare SHA1 hashs + chck_ptrn = message_recorder.find( + r"\* repo2 \[ [0-9a-f]{7} \] [0-9a-f]{7} \(missing upstream\)" + ) + pattern = re.compile("^.*repo2.*([0-9a-f]{7}).*([0-9a-f]{7}).*$") + if chck_ptrn: + restr = pattern.match(chck_ptrn) + if not (restr and restr.group(1) == restr.group(2)): + raise Exception("SHA1 does not match") + + assert message_recorder.find( + r"\* repo3 \[ point_c \] point_c \(expected: main\) \(missing upstream\)" + ) + assert message_recorder.find( + r"\* manifest \[ snapshot on moded \]= snapshot on moded \(dirty\) \(expected: master\) ~~ MANIFEST" # noqa: E501 + ) + + +def test_raw_on_update__check_for_remotes__warning( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + The goal is to hit "not useful Manifest" Warning, when on UPDATE + We have to use RAW dump here to simulate such Warning. To achieve + this we need to have some + Repo that does no have remotes (Grabed) and no remotes found on + UPDATE source (Deep Manifest) as well. This can only be achieved + by RAW dump. Thus output Manifest will print Warning. + + Note: this is not possible to hit when on Create from Workspace + as Workspace does not allow to have missing Remotes + + Scenario: + + * 1st: create bunch of repos + * 2nd: create Manifest repo + * 3rd: init Workspace + * 4th: delete remote from Deep Manifest + * 5th: check state by 'status' + * 6th: commit changes to Deep Manifest, so no (dirty) + * 7th: remove remote 'origin' from local Repo + * 8th: check 'status', it should report (missing remote) for current Repo state + * 9th: perform dump Workspace and UPDATE + * 10th: check if Warning is displayed + """ + + # 1st: create bunch of repos + git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "test.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + git_server.add_repo("repo3") + git_server.push_file("repo3", "test.txt") + git_server.add_repo("repo4") + git_server.push_file("repo4", "test.txt") + + # 2nd: create Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init Workspace + manifest_url = git_server.manifest_url + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: delete remote from Deep Manifest + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_file_path) + + # 5th: check state by 'status' + # it shourld report '(missing remote)' for Deep Manifest + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find(r"\* repo1-mr \[ master \(missing remote\) \] master") + assert message_recorder.find(r"\* repo2 \[ master \] master") + assert message_recorder.find( + r"\* manifest \[ master \]= master \(dirty\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo3 \[ master \] master") + assert message_recorder.find(r"\* repo4 \[ master \] master") + + # 6th: commit changes to Deep Manifest, so no (dirty) + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "adding repo without url") + run_git(manifest_path, "push", "-u", "origin", "master") + + # 7th: remove remote 'origin' from local Repo + repo1_mr_path = workspace_path / "repo1-mr" + run_git(repo1_mr_path, "remote", "remove", "origin") + + # 8th: check 'status', it should report (missing remote) for current Repo state + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"\* repo1-mr \[ master \(missing remote\) \] master \(missing remote\)" + ) + + # 9th: perform dump Workspace and UPDATE + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + + # 10th: check if Warning is displayed + assert message_recorder.find(r"=> UPDATING Deep Manifest on") # ... path + assert message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + assert message_recorder.find(r":: Dump complete") + + +def test_on_update__check_for_remotes__after_update_is_ok( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + The goal is to NOT hit "not useful Manifest" when on UPDATE + + Here we only delete 'url' from Deep Manifest. + That should gets UPDATEd back, so there should be no Warning + + Scenario: + + * 1st: create bunch of repos + * 2nd: create Manifest repo + * 3rd: init Workspace + * 4th: delete remote from Deep Manifest + * 5th: check state by 'status' + * 6th: commit changes to Deep Manifest, so no (dirty) + * 7th: check 'status', it should report (missing remote) for current Repo state + * 8th: perform dump Workspace and UPDATE + * 9th: check if Warning is displayed + """ + + # 1st: create bunch of repos + git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "test.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + git_server.add_repo("repo3") + git_server.push_file("repo3", "test.txt") + git_server.add_repo("repo4") + git_server.push_file("repo4", "test.txt") + + # 2nd: create Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init Workspace + manifest_url = git_server.manifest_url + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: delete remote from Deep Manifest + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_file_path) + + # 5th: check state by 'status' + # it shourld report '(missing remote)' for Deep Manifest + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find(r"\* repo1-mr \[ master \(missing remote\) \] master") + assert message_recorder.find(r"\* repo2 \[ master \] master") + assert message_recorder.find( + r"\* manifest \[ master \]= master \(dirty\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo3 \[ master \] master") + assert message_recorder.find(r"\* repo4 \[ master \] master") + + # 6th: commit changes to Deep Manifest, so no (dirty) + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "adding repo without url") + run_git(manifest_path, "push", "-u", "origin", "master") + + # 7th: check 'status', it should report (missing remote) for current Repo state + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"\* manifest \[ master \]= master ~~ MANIFEST" + ) + + # 8th: perform dump Workspace and UPDATE + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--update") + + # 9th: check if Warning is displayed + assert message_recorder.find(r"=> UPDATING Deep Manifest on") # ... path + assert not message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + assert message_recorder.find(r":: Dump complete") + + +def test_on_update__check_for_remotes__is_ok( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + this is successful version of: + 'test_on_update__check_for_remotes__warning' + just to see if Warning is not printed everytime + as that will defy other related tests + """ + + # 1st: create bunch of repos + git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "test.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + git_server.add_repo("repo3") + git_server.push_file("repo3", "test.txt") + git_server.add_repo("repo4") + git_server.push_file("repo4", "test.txt") + + # 2nd: create Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init Workspace + manifest_url = git_server.manifest_url + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: dump Workspace and UPDATE + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--update") + + # 5th: what we should not find is Warning + # like the one below + assert not message_recorder.find( + r"Warning: This Manifest is not useful due to some missing remotes" + ) + + +def test_on_update( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test actual outcome of the 'update' + + Scenario: + + * 1st: Create repositories + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: checkout, modify, commit and push Manifest repo to different branch + so we can keep Manifest with just 3 Repos + * 5th: add some other repos + * 6th: sync to update Workspace + now new repos is cloned to Workspace + * 7th: Manifest repo: go back to previous branch + so the current Manifest Repo + will be different from current statuses of the Workspace. + * 8th: update current Manifest integrated into Workspace + calling 'tsrc dump-manifest --update' + * 9th: check 'status' to know where we stand + Update: + * deleting: 'repo5' and 'repo6' + * updating: 'manifest' (changing branch) + * adding: 'repo3' and 'repo4' + """ + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + repo2_url = git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: checkout, modify, commit and push Manifest repo to different branch + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-B", "old_master") + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_insert_repo_item_to_manifest(manifest_file_path, repo2_url) + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "adding repos that does not exists") + run_git(manifest_path, "push", "-u", "origin", "old_master") + + # 5th: add some other repos + git_server.add_repo("repo3") + git_server.push_file("repo3", "test.txt") + git_server.add_repo("repo4") + git_server.push_file("repo4", "test.txt") + + # 6th: sync to update Workspace + tsrc_cli.run("sync") + + # 7th: Manifest repo: go back to previous branch + run_git(manifest_path, "checkout", "old_master") + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find(r"=> Destination \[Deep Manifest description\]") + assert message_recorder.find(r"\* repo3 master") + assert message_recorder.find(r"\* repo4 master") + assert message_recorder.find(r"\* repo2 \[ master \] master") + assert message_recorder.find(r"\* repo1 \[ master \] master") + assert message_recorder.find( + r"\* manifest \[ master \]= old_master \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"- repo5 \[ master \]") + assert message_recorder.find(r"- repo6 \[ master \]") + + # 8th: update current Manifest integrated into Workspace + tsrc_cli.run("dump-manifest", "--update") + + # 9th: check 'status' to know where we stand + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find(r"=> Destination \[Deep Manifest description\]") + # no ide what is wrong with below line + assert message_recorder.find( + r"\* manifest \[ old_master \]= old_master \(dirty\) \(expected: master\) ~~ MANIFEST" # noqa: E501 + ) + assert message_recorder.find(r"\* repo4 \[ master \] master") + assert message_recorder.find(r"\* repo2 \[ master \] master") + assert message_recorder.find(r"\* repo1 \[ master \] master") + assert message_recorder.find(r"\* repo3 \[ master \] master") + + +def ad_hoc_insert_repo_item_to_manifest(manifest_path: Path, some_url: str) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + yaml = ruamel.yaml.YAML(typ="rt") + parsed = yaml.load(manifest_path.read_text()) + for _, value in parsed.items(): + if isinstance(value, List): + value.append({"dest": "repo5", "url": some_url}) + value.append({"dest": "repo6", "url": some_url}) + + # write the file down + with open(manifest_path, "w") as file: + yaml.dump(parsed, file) diff --git a/tsrc/test/cli/test_dump_manifest__args.py b/tsrc/test/cli/test_dump_manifest__args.py new file mode 100644 index 00000000..fb1fc559 --- /dev/null +++ b/tsrc/test/cli/test_dump_manifest__args.py @@ -0,0 +1,667 @@ +import os +from pathlib import Path + +# import pytest +from cli_ui.tests import MessageRecorder + +from tsrc.git import run_git +from tsrc.manifest import load_manifest +from tsrc.test.cli.test_dump_manifest import ad_hoc_delete_remote_from_manifest +from tsrc.test.helpers.cli import CLI +from tsrc.test.helpers.git_server import GitServer +from tsrc.workspace_config import WorkspaceConfig + +""" +Contains: + +* test_wrong_args__raw_update__missing_workspace +* test_raw_dump_update__with_workspace__do_update__ok +* test_raw_dump_update__with_workspace__do_update__fail +* test_raw_dump_update__with_workspace__do_update__force +* test_raw_dump_update_with_multi_remotes__use_workspace__dirty__fail +* test_wrong_args__save_to_overwrite +* test_wrong_args__update_on_file +* test_wrong_args__update_and_update_on +* test_wrong_args__save_to_path +* test_wrong_args__update_on +* test_wrong_args__update +* test_wrong_args__save_to_update_on +""" + + +def test_raw_dump_update__with_workspace__do_update__ok( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Combination of RAW dump and Workspace Deep Manifest upate + + Option A: resonable and correct way + + Scenario: + + * 1st: Create repositories + * 2nd: add Manifest repository + * 3nd: init workspace on master + * 4th: create sub-dircetory for RAW dump (will be used later) + * 5th: introduce 'repo 3' ignoring Workpsace + * 6th: option A: RAW dump starting from '.' dir (reasonable) + """ + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: create sub-dircetory for RAW dump (will be used later) + sub1_path = "level 2" + os.makedirs(sub1_path) + + # 5th: introduce 'repo 3' ignoring Workpsace + os.chdir(sub1_path) + sub1_1_path = Path("repo 3") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 6th: option A: RAW dump starting from '.' dir (reasonable) + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update", "--preview") + + # 'repo 3' should be included in output + assert message_recorder.find(r"dest: manifest") + assert message_recorder.find( + r"level 2.*repo 3" + ), "Update on Deep Manifest was NOT successful" + + +def test_raw_dump_update__with_workspace__do_update__fail( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Combination of RAW dump and Workspace Deep Manifest upate + + Option B: not reasonable way, should FAIL + + Scenario: + + * 1st: Create repositories + * 2nd: add Manifest repository + * 3nd: init workspace on master + * 4th: create sub-dircetory for RAW dump (will be used later) + * 5th: introduce 'repo 3' ignoring Workpsace + * 6th: option B: RAW dump starting from 'level 2' dir (NOT reasonable) + """ + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: create sub-dircetory for RAW dump (will be used later) + sub1_path = "level 2" + os.makedirs(sub1_path) + + # 5th: introduce 'repo 3' ignoring Workpsace + os.chdir(sub1_path) + sub1_1_path = Path("repo 3") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 6th: option B: RAW dump starting from 'level 2' dir (NOT reasonable) + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", "level 2", "--update") + + assert message_recorder.find( + r"Error: Please consider again what you are trying to do." + ) + + +def test_raw_dump_update__with_workspace__do_update__force( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Combination of RAW dump and Workspace Deep Manifest upate + + Option C: using force (testing by load_manifest()) + + Scenario: + + * 1st: Create repositories + * 2nd: add Manifest repository + * 3nd: init workspace on master + * 4th: create sub-dircetory for RAW dump (will be used later) + * 5th: introduce 'repo 3' ignoring Workpsace + * 6th: option C: RAW dump starting from 'level 2' dir with '--force' + """ + + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + repo2_url = git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: create sub-dircetory for RAW dump (will be used later) + sub1_path = "level 2" + os.makedirs(sub1_path) + + # 5th: introduce 'repo 3' ignoring Workpsace + os.chdir(sub1_path) + sub1_1_path = Path("repo 3") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + run_git(full1_path, "remote", "add", "origin", repo2_url) + + # 6th: option C: RAW dump starting from 'level 2' dir with '--force' + os.chdir(workspace_path) + message_recorder.reset() + tsrc_cli.run( + "dump-manifest", + "--raw", + "level 2", + "--update", + "--force", + ) + + # 7th: check Manifest by load_manifest + is_fail: bool = False + is_ok: bool = False + w_m_path = workspace_path / "manifest" / "manifest.yml" + m = load_manifest(w_m_path) + for repo in m.get_repos(): + if repo.dest == "repo 3": + is_ok = True + # if repo.dest == "repo1" or repo.dest == "repo2" or repo.dest == "manifest": + if repo.dest in ["repo1", "repo2", "manifest"]: + is_fail = True + if is_fail is True or is_ok is False: + raise Exception("Found wrong repos in Manifest") + + +def test_raw_dump_update__repo_dirty__still_updates( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + when Deep Manifest is dirty, however + Manifest file 'manifest.yml' is clean, + allow update + + Scenario: + * 1st: Create repositories (1x multiple remotes) + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: DM: add file, so the repository will look dirty + * 5th: OK: as 'manifest.yml' is not dirty, allow update + """ + # 1st: Create repositories (1x multiple remotes) + # 'repo1-mr' will have multiple remotes + repo1_url = git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "CMakeLists.txt") + git_server.manifest.set_repo_remotes( + "repo1-mr", [("origin", repo1_url), ("upstream", "git@upstream.com")] + ) + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: DM: add file, so the repository will look dirty + Path(workspace_path / "manifest" / "dirty.test").touch() + run_git(workspace_path / "manifest", "add", "dirty.test") + + # 5th: OK: as 'manifest.yml' is not dirty, allow update + # even though repository in fact IS dirty + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + + assert message_recorder.find( + r"=> UPDATING Deep Manifest on '.*manifest.*manifest\.yml'" + ) + assert message_recorder.find(r":: Dump complete") + + +def test_raw_dump_update_with_multi_remotes__use_workspace__dirty__fail( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test if updating Deep Manifest is blocked + when its Repo is dirty + + this is done by early data check (before obtaining YAML data) + + Scenario: + * 1st: Create repositories (1x multiple remotes) + * 2nd: add Manifest repository + * 3rd: init workspace on master + * 4th: remove "origin" remote from 'manifest/manifest.yml" + * 5th: FAIL: does not allow update on dirty Deep Manifest + * 6th: no fail when using '--preview' + """ + # 1st: Create repositories (1x multiple remotes) + # 'repo1-mr' will have multiple remotes + repo1_url = git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "CMakeLists.txt") + git_server.manifest.set_repo_remotes( + "repo1-mr", [("origin", repo1_url), ("upstream", "git@upstream.com")] + ) + git_server.add_repo("repo2") + git_server.push_file("repo2", "test.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: remove "origin" remote from 'manifest/manifest.yml" + manifest_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_path) + + # 5th: FAIL: does not allow update on dirty Deep Manifest + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + assert message_recorder.find( + r"Error: not updating Deep Manifest as it is dirty, use '--force' to overide" + ) + + # 6th: no fail when using '--preview' + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update", "--preview") + assert message_recorder.find(r"dest: repo1-mr") + assert message_recorder.find(r"dest: repo2") + assert message_recorder.find(r"dest: manifest") + + +def test_wrong_args__save_to_overwrite( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if we get stopped when + '--save-to ' already exists + AND: + A) without '--force' => throws Error + B) with '--force' => should overwrite + C) wit '--preview' ==> should display + + Scenario + + * 1st: Create repositories, but not Manifest + * 2nd: init workspace on master + * 3rd: A) '--save-to': file exists + * 3rd: B) same, but now with '--force' + * 3rd: C) same, but now with '--preview' + """ + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test-manifest.yml") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: A) '--save-to': file exists + message_recorder.reset() + test_path_1: str = os.path.join("repo2", "test-manifest.yml") + tsrc_cli.run("dump-manifest", "--save-to", test_path_1) + assert message_recorder.find( + r"Error: 'SAVE_TO' file exist, use '--force' to overwrite existing file" + ) + + # 3rd: B) same, but now with '--force' + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--save-to", test_path_1, "--force") + assert message_recorder.find(r"=> OVERWRITING file '.*repo2.*test-manifest\.yml'") + assert message_recorder.find(r":: Dump complete") + + # 3rd: C) same, but now with '--preview' + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--save-to", test_path_1, "--preview") + assert message_recorder.find(r"dest: repo1") + assert message_recorder.find(r"dest: repo2") + + +def test_wrong_args__update_on__non_existent_file( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if we get stopped when + '--update-on ' when file does not exists + """ + + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: 'dump-manifest --update-on ' should fail + # if does not exist + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--update-on", "test-manifest.yml") + assert message_recorder.find(r"Error: 'UPDATE_AT' file does not exists") + + +def test_wrong_args__update_and_update_on( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if we get stopped when + '--update' and '--update-on ' is provided + """ + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: 'dump-manifest --update --update-on ' should fail + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--update", "--update-on", "test-manifest.yml") + assert message_recorder.find( + r"Error: Use only one out of '--update' or '--update-on' at a time" + ) + + +def test_wrong_args__save_to_path( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if when Workspace does not have Manifest repo, + the proper Error is reported + + Scenario: + * 1st: Create repositories, but not Manifest + * 2nd: init workspace on master + * 3rd: 'dump-manifest --save-to' should fail + as provided Path is not existing + """ + + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: 'dump-manifest --save-to' should fail + message_recorder.reset() + test_path_1: str = os.path.join("repo3", "manifest.yml") + tsrc_cli.run("dump-manifest", "--save-to", test_path_1) + assert message_recorder.find( + r"Error: 'SAVE_TO' directory structure must exists, however 'repo3' does not" + ) + + +def test_wrong_args__update_on( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if when Workspace does not have Manifest repo, + the proper Error is reported + + Scenario: + * 1st: Create repositories, but not Manifest + * 2nd: init workspace on master + * 3rd: 'dump-manifest --update-on' should fail + as there is no valid Manifest file there + """ + + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: 'dump-manifest --update-on' should fail + message_recorder.reset() + test_path_1: str = os.path.join("repo2", "CMakeLists.txt") + tsrc_cli.run("dump-manifest", "--update-on", test_path_1) + assert message_recorder.find(r"Error: Not able to load YAML data") + + +def test_wrong_args__raw_update__missing_workspace( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test if when on RAW dump, '--update' must found Workspace + (it should also be able to find DM, but that is tested elsewhere) + + Scenario: + + * 1st: create 'repo 1', GIT init, add, commit + * 2nd: create 'repo 2', GIT init, add, commit + * 3rd: RAW dump with updating DM, should fail + """ + + # 1st: create 'repo 1', GIT init, add, commit + os.chdir(workspace_path) + sub1_1_path = Path("repo 1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, workspace_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 2nd: create 'repo 2', GIT init, add, commit + os.chdir(workspace_path) + os.chdir(workspace_path) + sub1_2_path = Path("repo 2") + os.mkdir(sub1_2_path) + os.chdir(sub1_2_path) + full2_path: Path = Path(os.path.join(workspace_path, workspace_path, sub1_2_path)) + run_git(full2_path, "init") + sub1_2_1_file = Path("in_repo.txt") + sub1_2_1_file.touch() + run_git(full2_path, "add", "in_repo.txt") + run_git(full2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + + # 3rd: RAW dump with updating DM, should fail + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + assert message_recorder.find(r"Error: Could not find current workspace") + + +def test_wrong_args__update__no_dm( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test if when Workspace does not have Manifest repo, + the proper Error is reported + + Scenario: + * 1st: Create repositories, but not Manifest + * 2nd: init workspace on master + * 3rd: 'dump-manifest --update' should fail + as Manifest repository is not in the Workspace + """ + + # 1st: Create repositories, but not Manifest + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 3rd: 'dump-manifest --update' should fail + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--update") + assert message_recorder.find( + r"Error: Cannot obtain Deep Manifest from Workspace to update" + ) + + +def test_wrong_args__save_to_update_on( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + we should know, that Updating has higher importance, + therefore when '--save-to' and '--update-on' is used + at the same time, Warning should be shown about + 'SAVE_TO' to be ignored + + Scenario: + # 1st: Create repositories + # 2nd: add Manifest repository + # 3nd: init workspace on master + # 4th: change branch of Manifest's Repo + # 5th: dump-manifest with '--save-to' and '--update-on' + """ + + # 1st: Create repositories + git_server.add_repo("repo1") + git_server.push_file("repo1", "CMakeLists.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3nd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: change branch of Manifest's Repo + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-b", "dev") + + # 5th: dump-manifest with '--save-to' and '--update-on' + # should put a Warning about '--save-to' be ignored + # as updating has higher importance + message_recorder.reset() + test_path_1: str = os.path.join("repo2", "manifest.yml") + test_path_2: str = os.path.join("manifest", "manifest.yml") + tsrc_cli.run( + "dump-manifest", + "--save-to", + test_path_1, + "--update-on", + test_path_2, + ) + assert message_recorder.find( + r"Warning: 'SAVE_TO' path will be ignored when using '--update-on'" + ) diff --git a/tsrc/test/cli/test_dump_manifest__groups.py b/tsrc/test/cli/test_dump_manifest__groups.py new file mode 100644 index 00000000..6e1c074a --- /dev/null +++ b/tsrc/test/cli/test_dump_manifest__groups.py @@ -0,0 +1,617 @@ +""" +Test how Groups are taken care of when +'dump-manifest' command is in UPDATE mode + +as only than the Groups can be taken into account + +normal Repo grabbing (RAW or Workspace) cannot obtain +Groups. + +Someone may say, that Workspace Grab can work with Groups, +as they can be obtained from config. This is wrong as +such Groups does not contain items, therefore are +useles in this case. + +However Workspace config can be updated + +Contains: +* test_dump_manifest__constraints__no_remote_must_match_dest__on_update +* test_dump_manifest_workspace__update_with_constraints__add_repo +* test_dump_manifest_workspace__groups_delete +* test_dump_manifest__rename_repo +""" + +import os +from pathlib import Path +from typing import List + +# import pytest +import ruamel.yaml +from cli_ui.tests import MessageRecorder + +from tsrc.dump_manifest import ManifestDumper +from tsrc.dump_manifest_helper import MRISHelpers +from tsrc.git import run_git +from tsrc.manifest import Manifest, load_manifest, load_manifest_safe_mode +from tsrc.manifest_common_data import ManifestsTypeOfData +from tsrc.repo import Remote, Repo +from tsrc.test.cli.test_dump_manifest import ad_hoc_delete_remote_from_manifest +from tsrc.test.helpers.cli import CLI +from tsrc.test.helpers.git_server import GitServer + +# from tsrc.test.helpers.message_recorder_ext import MessageRecorderExt + + +def test_dump_manifest__constraints__no_remote_must_match_dest__on_update( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + if we are performing UPDATE on Manifest and some Repo + does not have remotes (so it is hard to identify it) + we should use 'dest' instead and update it accordingly + + Scenario: + + * 1st: add few Repos + * 2nd: add Manifest repository + * 3rd: add Groups (to test constraints) + * 4th: init workspace on master + * 5th: delete remote of one Repo of Deep Manifest + * 6th: commit and push Deep Manifest + * 7th: create new manifest providing different Groups + * 8th: verify Manifest created by Group 1 + * 9th: verify Manifest created by Group 2 + """ + # 1st: add few Repos + git_server.add_repo("repo1-mr") + git_server.push_file("repo1-mr", "test_1-mr.txt") + git_server.add_repo("repo2") + git_server.push_file("repo2", "test2.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: add Groups (to test constraints) + git_server.add_group("group_1", ["manifest", "repo1-mr"]) + git_server.add_group("group_2", ["manifest", "repo2"]) + + # 4th: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + + # 5th: delete remote of one Repo of Deep Manifest + manifest_path_file = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_remote_from_manifest(manifest_path_file) + + # 6th: commit and push Deep Manifest + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "adding repo without url") + run_git(manifest_path, "push", "-u", "origin", "master") + + # 7th: create new manifest providing different Groups + tsrc_cli.run( + "dump-manifest", "--update", "--save-to", "manifest_g_1.yml", "-g", "group_1" + ) + tsrc_cli.run( + "dump-manifest", "--update", "--save-to", "manifest_g_2.yml", "-g", "group_2" + ) + + # 8th: verify Manifest created by Group 1 + # it should update remote on 'repo1-mr' + m_1_file = workspace_path / "manifest_g_1.yml" + if m_1_file.is_file() is False: + raise Exception("Manifest file does not exists") + # m_1 = load_manifest_safe_mode(m_1_file, ManifestsTypeOfData.SAVED) + m_1 = load_manifest(m_1_file) # this Manifest should be fine + count: int = 0 + for repo in m_1.get_repos(): + if repo.dest == "repo1-mr": + if repo.remotes: + count += 1 + elif repo.dest == "repo2": + count += 2 + elif repo.dest == "manifest": + count += 4 + else: + raise Exception("Manifest contain wrong item") + if count != 7: + raise Exception("Manifest does not contain all items") + + # 9th: verify Manifest created by Group 2 + # it should NOT update remote on 'repo1-mr' + m_2_file = workspace_path / "manifest_g_2.yml" + if m_2_file.is_file() is False: + raise Exception("Manifest file 2 does not exists") + m_2 = load_manifest_safe_mode(m_2_file, ManifestsTypeOfData.SAVED) + count = 0 + for repo in m_2.get_repos(): + if repo.dest == "repo1-mr": + if not repo.remotes: + count += 1 + elif repo.dest == "repo2": + count += 2 + elif repo.dest == "manifest": + count += 4 + else: + raise Exception("Manifest contain wrong item") + if count != 7: + raise Exception("Manifest does not contain all items") + + +def test_dump_manifest_workspace__update_with_constraints__add_repo( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test selective Adding new Repo when on + RAW dump + UPDATE mode + user-provided Groups + (thus with constraints) + + Story: + + when using RAW dump and UPDATE and we want + to limit such update on selected Group(s), + then other Repos that does not match such + Group(s) should be left alone (not updated). + However when there is Repo in Manifest we + are updating, that is not found in such Manifest, + and at the same time was found by RAW dump, + this is sign that such Repo should be Added + (since it is clearly missing from Manifest) + Hovever if we did not use such Group, when + such Repo is included, there should be no Addition + + Scenario: + + * 1st: Create bunch of repos + * 2nd: add Manifest repo + * 3rd: add bunch of Groups + * 4th: init workspace with only selected group + * 5th: push current manifest as git branch "working" + * 6th: go back to Manifest's Git branch "master" + * 7th: adding Group, with non-existant Repo dest + * 8th: ugly sync, as there is non-existant Repo 'repo_3x' as item in Group + * 9th: change back to "working" branch and 'sync' + * 10th: now go to Manifest with 'repo_3x' in Groups + * 11th: ad-hoc create 'repo_3x', GIT init, add, commit, set remote + * 12th: RAW dump when with Group 'group_1' + * 13th: checking if 'repo_3x' is NOT there + * 14th: another RAW dump when with Group 'group_3' + * 15th: checking if 'repo_3x' is there, if not, it is FAIL + """ + # 1st: Create bunch of repos + git_server.add_repo("repo_1") + git_server.push_file("repo_1", "my_file_in_repo_1.txt") + git_server.add_repo("repo_2") + git_server.push_file("repo_2", "my_file_in_repo_2.txt") + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "my_file_in_repo_3.txt") + repo_4_url = git_server.add_repo("repo_4") + git_server.push_file("repo_4", "my_file_in_repo_4.txt") + + # 2nd: add Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + manifest_url = git_server.manifest_url + + # 3rd: add bunch of Groups + git_server.add_group("group_1", ["manifest", "repo_1", "repo_3"]) + git_server.add_group("group_3", ["manifest", "repo_3"]) + + # 4th: init workspace with only selected group + tsrc_cli.run( + "init", + "--branch", + "master", + manifest_url, + "--clone-all-repos", + "--groups", + "group_1", + "group_3", + ) + + # 5th: push current manifest as git branch "working" + manifest_path = workspace_path / "manifest" + manifest_path_file = workspace_path / "manifest" / "manifest.yml" + run_git(manifest_path, "checkout", "-b", "working") + ad_hoc_update_manifest_repo(manifest_path_file, manifest_url, "working") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new_manifest") + run_git(manifest_path, "push", "-u", "origin", "working") + + # 6th: go back to Manifest's Git branch "master" + run_git(manifest_path, "checkout", "master") + + # 7th: adding Group, with non-existant Repo dest + # we will return to this later + git_server.add_group( + "group_3", ["manifest", "repo_3", "repo_3x"], do_add_repos=False + ) + + # 8th: ugly sync, as there is non-existant Repo 'repo_3x' as item in Group + tsrc_cli.run("sync", "--ignore-missing-groups", "--ignore-missing-group-items") + + # 9th: change back to "working" branch and 'sync' + # still we have to ignore missing group item as there is still 'repo_3x' + # and to avoid further Errors on reading Deep Manifest + # and after the branch change even an Error on Future Manifest, + # we should disable thouse as well ('--no-dm', '--no-fm') + tsrc_cli.run( + "manifest", + "--ignore-missing-group-items", + "--no-dm", + "--no-fm", + "--branch", + "working", + ) + tsrc_cli.run("sync") + + # 10th: now go to Manifest with 'repo_3x' in Groups + # We are now in clean working state, no missing Group item + # is present, however if checkout "master" branch of Manifest + # we will introduce non-existant Repo 'repo_3x' again, + # this time only to Deep Manifest. + # Now when we call RAW dump + UPDATE + Group constraints + # when our Group selection will contain 'group_3' where + # the 'repo_3x' is included, than only in this case + # the 'dump-manifest' will add this Repo when on Group constraints + run_git(manifest_path, "checkout", "master") + + # 11th: ad-hoc create 'repo_3x', GIT init, add, commit, set remote + # So from this point forward, the 'repo_3x' exists. + # It is not present in the Workspace, but we will be using + # RAW dump anyway + sub1_1_path = Path("repo_3x") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full1_path: Path = Path(os.path.join(workspace_path, sub1_1_path)) + run_git(full1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full1_path, "add", "in_repo.txt") + run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + run_git(full1_path, "remote", "add", "origin", repo_4_url) + + # 12th: RAW dump when with Group 'group_1' + # First, let us try how RAW dump + UPDATE + Group constraints + # DOES NOT add 'repo_3x' as it is not included in 'group_1' items. + os.chdir(workspace_path) + tsrc_cli.run( + "dump-manifest", + "--raw", + ".", + "--update", + "--save-to", + "manifest_g_1.yml", + "-g", + "group_1", + ) + + # 13th: checking if 'repo_3x' is NOT there + w_m_path = workspace_path / "manifest_g_1.yml" + m = load_manifest_safe_mode(w_m_path, ManifestsTypeOfData.LOCAL) + for repo in m.get_repos(): + if repo.dest == "repo_3x": + raise Exception("failed Manifest update on repos of Group 'group_1'") + + # 14th: another RAW dump when with Group 'group_3' + # Here in Group 'group_3', the 'repo_3x' IS PRESENT + # therefore RAW dump + UPDATE + Group constraints + # should add it as new item to Repos + tsrc_cli.run( + "dump-manifest", + "--raw", + ".", + "--update", + "--save-to", + "manifest_g_3.yml", + "-g", + "group_3", + ) + + # 15th: checking if 'repo_3x' is there, if not, it is FAIL + w_m_path = workspace_path / "manifest_g_3.yml" + m = load_manifest_safe_mode(w_m_path, ManifestsTypeOfData.LOCAL) + match_repo_3x: bool = False + for repo in m.get_repos(): + if repo.dest == "repo_3x": + match_repo_3x = True + if match_repo_3x is False: + raise Exception("failed Manifest update on repos of Group 'group_3'") + + +def test_dump_manifest_workspace__groups_delete( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + # TODO: test it also when cloning all Repos + # this should include 'repo_4' as well + """ + when deleting Repos, it should also be deleted + from Group's 'repos:'. this test verifies that + + Scenario: + + * 1st: Create bunch of repos + * 2nd: add Manifest repo + * 3rd: add bunch of Groups + * 4th: init workspace with only selected group + * 5th: edit Manifest: remove Repo + * 6th: checkout new branch, commit, push Manifest + * 7th: tsrc sync new branch + * 8th: checkout manifest branch back + * 9th: tsrc dump-manifest '--update' + * 10th: use 'load_manifest to verify Deep Manfiest + """ + + # 1st: Create bunch of repos + git_server.add_repo("repo_1") + git_server.push_file("repo_1", "my_file_in_repo_1.txt") + git_server.add_repo("repo_2") + git_server.push_file("repo_2", "my_file_in_repo_2.txt") + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "my_file_in_repo_3.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "my_file_in_repo_4.txt") + + # 2nd: add Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + manifest_url = git_server.manifest_url + + # 3rd: add bunch of Groups + git_server.add_group("group_1", ["manifest", "repo_1", "repo_3"]) + git_server.add_group("group_3", ["repo_3"]) + + # 4th: init workspace with only selected group + tsrc_cli.run( + "init", "--branch", "master", manifest_url, "--groups", "group_1", "group_3" + ) + + # 5th: edit Manifest: remove Repo + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_repo_from_manifest(manifest_file_path) + + # 6th: checkout new branch, commit, push Manifest + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-b", "without_3") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "without_3") + run_git(manifest_path, "push", "-u", "origin", "without_3") + + # 7th: tsrc sync new branch + tsrc_cli.run("manifest", "-b", "without_3") + tsrc_cli.run("sync") + + # 8th: checkout manifest branch back + run_git(manifest_path, "checkout", "master") + + # 9th: tsrc dump-manifest '--update' + tsrc_cli.run("dump-manifest", "--update") + # so it should delete from 'manifest.yml' + # and thus delete from Groups as well + + # 10th: use 'load_manifest to verify Deep Manfiest + # 'repo_3' should not be there anymore + # not even amont the group's elements + w_m_path = workspace_path / "manifest" / "manifest.yml" + m = load_manifest(w_m_path) + if m.group_list: + for element in m.group_list.all_elements: + if element == "repo_3": + raise Exception("failed Manifest update on group items") + for repo in m.get_repos(): + if repo.dest == "repo_3": + raise Exception("failed Manifest update on repos") + + +def test_dump_manifest__rename_repo( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test feature for Rename Repos's dest + it also renames Repo's dest of Groups + + Scenario + + * 1st: Create bunch of repos + * 2nd: add Manifest repo + * 3rd: add bunch of Groups + * 4th: init workspace with only selected group + * 5th: edit Manifest: remove Repo + * 6th: checkout new branch, commit, push Manifest + * 7th: tsrc sync new branch + * 8th: checkout manifest branch back + * 9th: rename bunch of Repos's dest + * 10th: tsrc dump-manifest RAW '--update' + * 11th: verify by load_manifest: Groups.elements + * 12th: verify by load_manifest: Repos's dest vs URL + """ + + # 1st: Create bunch of repos + # also: intentionaly put there 'repo_4-7509f8e' + # as it is exact first replacement name for + # 'repo_4' and thus also test generation + # of new replacement name + repo_1_url = git_server.add_repo("repo_1") + git_server.push_file("repo_1", "my_file_in_repo_1.txt") + git_server.add_repo("repo_2") + git_server.push_file("repo_2", "my_file_in_repo_2.txt") + repo_3_url = git_server.add_repo("repo_3") + git_server.push_file("repo_3", "my_file_in_repo_3.txt") + repo_4_url = git_server.add_repo("repo_4") + git_server.push_file("repo_4", "my_file_in_repo_4.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "my_file_in_repo_4.txt") + git_server.add_repo("repo_4-7509f8e") + git_server.push_file("repo_4-7509f8e", "my_file_in_repo_4.txt") + + # 2nd: add Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + manifest_url = git_server.manifest_url + + # 3rd: add bunch of Groups + git_server.add_group("group_1", ["manifest", "repo_1", "repo_3"]) + git_server.add_group("group_3", ["repo_3"]) + + # 4th: init workspace with only selected group + tsrc_cli.run( + "init", + "--branch", + "master", + manifest_url, + "--clone-all-repos", + "--groups", + "group_1", + "group_3", + ) + + # 5th: edit Manifest: remove Repo + manifest_file_path = workspace_path / "manifest" / "manifest.yml" + ad_hoc_delete_repo_from_manifest(manifest_file_path) + + # 6th: checkout new branch, commit, push Manifest + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-b", "without_3") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "without_3") + run_git(manifest_path, "push", "-u", "origin", "without_3") + + # 7th: tsrc sync new branch + tsrc_cli.run("manifest", "-b", "without_3") + tsrc_cli.run("sync") + + # 8th: checkout manifest branch back + run_git(manifest_path, "checkout", "master") + + # 9th: rename bunch of Repos's dest + os.rename("repo_1", "repo_1_renamed") + os.rename("repo_4", "repo_1") + os.rename("repo_1_renamed", "repo_4") + # one more without colision + os.rename("repo_3", "repo_3x") + + # 10th: tsrc dump-manifest RAW '--update' + tsrc_cli.run("dump-manifest", "--raw", ".", "--update") + + # 11th: verify by load_manifest: Groups.elements + m = load_manifest(manifest_file_path) + _part_11(m) + + # 12th: verify by load_manifest: Repos's dest vs URL + _part_12(m, repo_1_url, repo_3_url, repo_4_url) + + +def _part_11(m: Manifest) -> None: + is_ok: int = 0 + if m.group_list and m.group_list.groups: + for g in m.group_list.groups: + for ge in m.group_list.groups[g].elements: + if g == "group_1": + if ge == "manifest": + is_ok += 1 + elif ge == "repo_4": + is_ok += 2 + elif ge == "repo_3x": + is_ok += 4 + else: + is_ok = -1000 + elif g == "group_3": + if ge == "repo_3x": + is_ok += 8 + else: + is_ok = -1000 + if is_ok != 15: + raise Exception("Manifest's Groups items mismach") + + +# flake8: noqa: C901 +def _part_12(m: Manifest, repo_1_url: str, repo_3_url: str, repo_4_url: str) -> None: + is_ok: int = 0 + repos = m.get_repos(all_=True) + for repo in repos: + if repo.dest == "repo_1" and repo.clone_url == repo_4_url: + is_ok += 1 + elif repo.dest == "repo_3x" and repo.clone_url == repo_3_url: + is_ok += 2 + elif repo.dest == "repo_4" and repo.clone_url == repo_1_url: + is_ok += 4 + elif repo.dest == "repo_2": + is_ok += 8 + elif repo.dest == "repo_5": + is_ok += 16 + elif repo.dest == "repo_4-7509f8e": + is_ok += 32 + elif repo.dest == "manifest": + is_ok += 64 + else: + is_ok = -1000 + if is_ok != 127: + raise Exception("Manifest's Repo mismatch") + + +def ad_hoc_delete_repo_from_manifest( + manifest_path: Path, +) -> None: + """ + this time we will call function + that deletes from manifest + """ + manifest_path.parent.mkdir(parents=True, exist_ok=True) + yaml = ruamel.yaml.YAML(typ="rt") + y = yaml.load(manifest_path.read_text()) + + del_list: List[str] = ["repo_3"] + + is_updated_tmp: List[bool] = [False] # dummy + m_d = ManifestDumper() + m_d._walk_yaml_delete_group_items(y, 0, False, False, del_list, is_updated_tmp) + m_d._walk_yaml_delete_repos_items(y, 0, False, del_list, is_updated_tmp) + + # write the file down + with open(manifest_path, "w") as file: + yaml.dump(y, file) + + +def ad_hoc_update_manifest_repo( + manifest_path: Path, + manifest_url: str, + manifest_branch: str, +) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + yaml = ruamel.yaml.YAML(typ="rt") + y = yaml.load(manifest_path.read_text()) + + u_m_list: List[str] = ["manifest"] + + repos: List[Repo] = [ + Repo( + dest="manifest", + remotes=[Remote(name="origin", url=manifest_url)], + branch=manifest_branch, + ) + ] + + mris_h = MRISHelpers(repos=repos) + mris = mris_h.mris + + is_updated_tmp: List[bool] = [False] # dummy + m_d = ManifestDumper() + m_d._walk_yaml_update_repos_items(y, 0, mris, False, u_m_list, is_updated_tmp) + + # write the file down + with open(manifest_path, "w") as file: + yaml.dump(y, file) diff --git a/tsrc/test/cli/test_groups_extra.py b/tsrc/test/cli/test_groups_extra.py index fbcfa977..245e06a1 100644 --- a/tsrc/test/cli/test_groups_extra.py +++ b/tsrc/test/cli/test_groups_extra.py @@ -724,8 +724,7 @@ def ad_hoc_update_dm_branch( if isinstance(value, List): for x in value: if isinstance(x, ruamel.yaml.comments.CommentedMap): - # if x["dest"] == "repo_1": - if x["dest"] == "manifest": + if "dest" in x and x["dest"] == "manifest": x["branch"] = "br" # write the file down diff --git a/tsrc/test/cli/test_sync_extended.py b/tsrc/test/cli/test_sync_extended.py index 2cf0a216..1305193a 100644 --- a/tsrc/test/cli/test_sync_extended.py +++ b/tsrc/test/cli/test_sync_extended.py @@ -388,7 +388,7 @@ def test_sync_on_groups_intersection__case_3_b( # but with '--ignore-missing-groups' thus it will be ignored # from provided list, and config will be updated only with # proper Groups - # (skiping 'group_5' as it was not entered in '--groups') + # (skipping 'group_5' as it was not entered in '--groups') tsrc_cli.run( "sync", "--ignore-missing-groups", "--groups", "group_1", "group_2", "group_3" ) diff --git a/tsrc/test/cli/test_sync_to_ref.py b/tsrc/test/cli/test_sync_to_ref.py index 33c6c8e7..6a42148d 100644 --- a/tsrc/test/cli/test_sync_to_ref.py +++ b/tsrc/test/cli/test_sync_to_ref.py @@ -387,11 +387,12 @@ def ad_hoc_update_dm_repo_branch_and_sha1( if isinstance(value, List): for x in value: if isinstance(x, ruamel.yaml.comments.CommentedMap): - if x["dest"] == "main-proj-backend": - x["branch"] = "dev" - x["sha1"] = devs_sha1 - if x["dest"] == "manifest": - x["branch"] = "cmp-1" + if "dest" in x: + if x["dest"] == "main-proj-backend": + x["branch"] = "dev" + x["sha1"] = devs_sha1 + if x["dest"] == "manifest": + x["branch"] = "cmp-1" # write the file down with open(manifest_path, "w") as file: yaml.dump(parsed, file) @@ -411,11 +412,12 @@ def ad_hoc_update_dm_repo_branch_and_tag( if isinstance(value, List): for x in value: if isinstance(x, ruamel.yaml.comments.CommentedMap): - if x["dest"] == "main-proj-backend": - x["branch"] = "dev" - x["tag"] = this_tag - if x["dest"] == "manifest": - x["branch"] = "cmp-1" + if "dest" in x: + if x["dest"] == "main-proj-backend": + x["branch"] = "dev" + x["tag"] = this_tag + if x["dest"] == "manifest": + x["branch"] = "cmp-1" # write the file down with open(manifest_path, "w") as file: yaml.dump(parsed, file) diff --git a/tsrc/test/conftest.py b/tsrc/test/conftest.py index a209f6e2..28bdc20d 100644 --- a/tsrc/test/conftest.py +++ b/tsrc/test/conftest.py @@ -9,6 +9,7 @@ from tsrc.test.helpers.cli import tsrc_cli # noqa: F401 from tsrc.test.helpers.git_server import git_server # noqa: F401 +from tsrc.test.helpers.message_recorder_ext import MessageRecorderExt from tsrc.workspace import Workspace @@ -41,3 +42,11 @@ def message_recorder() -> Iterator[MessageRecorder]: res.start() yield res res.stop() + + +@pytest.fixture +def message_recorder_ext(request: Any) -> Iterator[MessageRecorderExt]: + recorder = MessageRecorderExt() + recorder.start() + yield recorder + recorder.stop() diff --git a/tsrc/test/helpers/git_server.py b/tsrc/test/helpers/git_server.py index 714496ea..a9545ec2 100644 --- a/tsrc/test/helpers/git_server.py +++ b/tsrc/test/helpers/git_server.py @@ -203,7 +203,7 @@ def configure_group( def get_repo(self, name: str) -> RepoConfig: for repo in self.data["repos"]: - if repo["dest"] == name: + if "dest" in repo and repo["dest"] == name: return cast(RepoConfig, repo) raise AssertionError(f"repo '{name}' not found in manifest") @@ -340,15 +340,18 @@ def add_manifest_repo( self.manifest.add_repo(name, url, branch=default_branch) return url - def add_group(self, group_name: str, repos: List[str]) -> None: + def add_group( + self, group_name: str, repos: List[str], do_add_repos: bool = True + ) -> None: """ adding new group should not be blocked when repositories inside are already exists. """ - for repo in repos: - repo_path = self.bare_path / repo - if not repo_path.exists(): - self.add_repo(repo) + if do_add_repos is True: + for repo in repos: + repo_path = self.bare_path / repo + if not repo_path.exists(): + self.add_repo(repo) self.manifest.configure_group(group_name, repos) def push_file( diff --git a/tsrc/test/helpers/manifest_file.py b/tsrc/test/helpers/manifest_file.py index 95ad1edb..150c02c2 100644 --- a/tsrc/test/helpers/manifest_file.py +++ b/tsrc/test/helpers/manifest_file.py @@ -19,7 +19,7 @@ def ad_hoc_deep_manifest_manifest_branch( if isinstance(value, List): for x in value: if isinstance(x, ruamel.yaml.comments.CommentedMap): - if x["dest"] == "manifest": + if "dest" in x and x["dest"] == "manifest": x.insert(2, "branch", branch) with open(manifest_path, "w") as file: @@ -38,7 +38,7 @@ def ad_hoc_deep_manifest_manifest_url( if isinstance(value, List): for x in value: if isinstance(x, ruamel.yaml.comments.CommentedMap): - if x["dest"] == "manifest": + if "dest" in x and x["dest"] == "manifest": x["url"] = url with open(manifest_path, "w") as file: diff --git a/tsrc/test/helpers/message_recorder_ext.py b/tsrc/test/helpers/message_recorder_ext.py new file mode 100644 index 00000000..215f6f41 --- /dev/null +++ b/tsrc/test/helpers/message_recorder_ext.py @@ -0,0 +1,76 @@ +""" +this code extend the one in: +'https://github.com/your-tools/python-cli-ui' + +and it is 'find_next' function that is added +(implemented here) + +patch was provided to original source in a form +of [pull request #115](https://github.com/your-tools/python-cli-ui/pull/115) +""" + +import re +from typing import Any, Iterator, Optional + +import cli_ui +import pytest + + +class MessageRecorderExt: + """Helper class to tests emitted messages""" + + def __init__(self) -> None: + cli_ui._MESSAGES = [] + self.idx_find_next: int = 0 + + def start(self) -> None: + """Start recording messages""" + cli_ui.CONFIG["record"] = True + + def stop(self) -> None: + """Stop recording messages""" + cli_ui.CONFIG["record"] = False + cli_ui._MESSAGES = [] + + def reset(self) -> None: + """Reset the list""" + cli_ui._MESSAGES = [] + + def find(self, pattern: str) -> Optional[str]: + """Find a message in the list of recorded message + + :param pattern: regular expression pattern to use + when looking for recorded message + """ + regexp = re.compile(pattern) + for idx, message in enumerate(cli_ui._MESSAGES): + if re.search(regexp, message): + if isinstance(message, str): + self.idx_find_next = idx + 1 + return message + return None + + def find_right_after(self, pattern: str) -> Optional[str]: + """Same as 'find', but only check the message that is right after + the one found last time. if no message was found before, the 1st + message in buffer is checked + + This is particulary usefull if we want to match only consecutive message. + Calling this function can be repeated for further consecutive message match. + """ + if len(cli_ui._MESSAGES) > self.idx_find_next: + regexp = re.compile(pattern) + message = cli_ui._MESSAGES[self.idx_find_next] + if re.search(regexp, message): + if isinstance(message, str): + self.idx_find_next += 1 + return message + return None + + +@pytest.fixture +def message_recorder_ext(request: Any) -> Iterator[MessageRecorderExt]: + recorder = MessageRecorderExt() + recorder.start() + yield recorder + recorder.stop() diff --git a/tsrc/workspace.py b/tsrc/workspace.py index 05ca5cbd..a2a1ecb4 100644 --- a/tsrc/workspace.py +++ b/tsrc/workspace.py @@ -14,6 +14,7 @@ from tsrc.git import is_git_repository from tsrc.local_manifest import LocalManifest from tsrc.manifest import Manifest +from tsrc.manifest_common_data import ManifestsTypeOfData from tsrc.remote_setter import RemoteSetter from tsrc.repo import Repo from tsrc.syncer import Syncer @@ -66,6 +67,11 @@ def __init__(self, root_path: Path) -> None: def get_manifest(self) -> Manifest: return self.local_manifest.get_manifest() + def get_manifest_safe_mode(self, mtod: ManifestsTypeOfData) -> Manifest: + return self.local_manifest.get_manifest_safe_mode( + mtod, + ) + def update_manifest(self) -> None: manifest_url = self.config.manifest_url manifest_branch = self.config.manifest_branch @@ -74,12 +80,19 @@ def update_manifest(self) -> None: self.local_manifest.update(url=manifest_url, branch=manifest_branch) - def update_config_repo_groups(self, groups: Optional[List[str]]) -> None: + def update_config_repo_groups( + self, groups: Optional[List[str]], ignore_group_item: bool = False + ) -> None: if groups: self.config.repo_groups = groups self.config.save_to_file(self.cfg_path) else: - local_manifest = self.local_manifest.get_manifest() + if ignore_group_item is True: + local_manifest = self.local_manifest.get_manifest_safe_mode( + ManifestsTypeOfData.LOCAL + ) + else: + local_manifest = self.local_manifest.get_manifest() if local_manifest.group_list: self.config.repo_groups = list(local_manifest.group_list.groups) self.config.save_to_file(self.cfg_path) @@ -119,11 +132,16 @@ def set_remotes(self, num_jobs: int = 1) -> None: raise RemoteSetterError def perform_filesystem_operations( - self, manifest: Optional[Manifest] = None + self, + manifest: Optional[Manifest] = None, + ignore_group_item: bool = False, ) -> None: repos = self.repos if not manifest: - manifest = self.get_manifest() + if ignore_group_item is True: + manifest = self.get_manifest_safe_mode(ManifestsTypeOfData.LOCAL) + else: + manifest = self.get_manifest() operator = FileSystemOperator(self.root_path, repos) operations = manifest.file_system_operations known_repos = [x.dest for x in repos] diff --git a/tsrc/workspace_repos_summary.py b/tsrc/workspace_repos_summary.py index 48a4729e..37925bea 100644 --- a/tsrc/workspace_repos_summary.py +++ b/tsrc/workspace_repos_summary.py @@ -23,7 +23,7 @@ from tsrc.local_manifest import LocalManifest from tsrc.manifest import Manifest, RepoNotFound from tsrc.manifest_common import ManifestGetRepos, ManifestGroupNotFound -from tsrc.manifest_common_data import ManifestsTypeOfData, get_main_color +from tsrc.manifest_common_data import ManifestsTypeOfData, mtod_get_main_color from tsrc.pcs_repo import PCSRepo from tsrc.repo import Repo from tsrc.status_endpoint import Status @@ -918,7 +918,11 @@ def _describe_deep_manifest( ) -> List[ui.Token]: message: List[ui.Token] = [] if d_m_r_found is True and isinstance(d_m_repo, Repo): - message += [get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "[", ui.green] + message += [ + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + "[", + ui.green, + ] desc, _ = d_m_repo.describe_to_tokens( self.max_dm_desc, ManifestsTypeOfData.DEEP ) @@ -926,26 +930,26 @@ def _describe_deep_manifest( if sm and dest == sm.dest: if self.d_m_root_point is True: message += [ - get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "]=", ui.reset, ] else: message += [ - get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "]", ui.reset, ] else: if self.d_m_root_point is True: message += [ - get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "] ", ui.reset, ] else: message += [ - get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "]", ui.reset, ] @@ -993,7 +997,7 @@ def _describe_status_apprise_part( ) -> List[ui.Token]: """usefull for Future Manifest""" message: List[ui.Token] = [] - message += [get_main_color(ManifestsTypeOfData.FUTURE)] + message += [mtod_get_main_color(ManifestsTypeOfData.FUTURE)] message += ["("] if apprise_repo: desc, desc_cmp = apprise_repo.describe_to_tokens( @@ -1008,7 +1012,7 @@ def _describe_status_apprise_part( if self.lfm and self.max_fm_desc > 0: message += [" ".ljust(self.max_fm_desc), "<<"] message += desc_tokens - message += [get_main_color(ManifestsTypeOfData.FUTURE)] + message += [mtod_get_main_color(ManifestsTypeOfData.FUTURE)] message += [")", ui.reset] return message @@ -1022,7 +1026,7 @@ def _describe_on_manifest( align_before: int = 0, ) -> List[ui.Token]: message: List[ui.Token] = [] - message += [get_main_color(tod)] + message += [mtod_get_main_color(tod)] l_just = 0 if tod == ManifestsTypeOfData.DEEP: @@ -1193,23 +1197,31 @@ def _describe_deep_manifest_leftovers_repo_dm_column( self, leftover: Repo ) -> List[ui.Token]: message: List[ui.Token] = [] - message += [get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "["] - message += [get_main_color(ManifestsTypeOfData.DEEP)] + message += [mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "["] + message += [mtod_get_main_color(ManifestsTypeOfData.DEEP)] desc, _ = leftover.describe_to_tokens( self.max_dm_desc, ManifestsTypeOfData.DEEP ) message += desc if self.d_m_root_point is True: - message += [get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "] ", ui.reset] + message += [ + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + "] ", + ui.reset, + ] else: - message += [get_main_color(ManifestsTypeOfData.DEEP_BLOCK), "]", ui.reset] + message += [ + mtod_get_main_color(ManifestsTypeOfData.DEEP_BLOCK), + "]", + ui.reset, + ] return message def _describe_leftover_repo_dest_column( self, leftover: Repo, tod: ManifestsTypeOfData ) -> List[ui.Token]: message: List[ui.Token] = [] - main_color = get_main_color(tod) + main_color = mtod_get_main_color(tod) if (self.workspace.root_path / leftover.dest).is_dir() is True: if leftover.dest in self.leftover_statuses: status = self.leftover_statuses[leftover.dest]