diff --git a/gitlab_sync/config.py b/gitlab_sync/config.py index 8b5d289..97c6973 100644 --- a/gitlab_sync/config.py +++ b/gitlab_sync/config.py @@ -54,6 +54,22 @@ def valid_strategy(value: str) -> typing.Callable[["RunConfig"], None]: return strategy +""" +Grand config hierachy: +class Config: + concurrency = 1 + sync_configs = [SyncConfig] + +class Sync: + *config + repositories + + +Sync: + config = SyncConfig + repos = [Repository] + +""" schema = Schema({ Required(absolute_dir_path): { Required(All("access-token", Replace("-", "_"))): And( diff --git a/gitlab_sync/operations.py b/gitlab_sync/operations.py index 64d731d..18ae8be 100644 --- a/gitlab_sync/operations.py +++ b/gitlab_sync/operations.py @@ -15,13 +15,14 @@ def clone(config, repo): """Clone a new repository.""" os.makedirs(str(repo.local_path)) repo.git("init", ".") - repo.git("config", "--local", "--add", "gitlab-sync.project-id", str(repo.id)) + repo.git("config", "--local", "gitlab-sync.project-id", str(repo.id)) repo.git("remote", "add", "origin", config.gitlab_git + "%s.git" % repo.gitlab_path) update_local(repo) def update_local(repo): """Update master from the remote.""" + repo.git("config", "--local", "gitlab-sync.project-path", str(repo.gitlab_path)) repo.git("fetch") # get refs/remotes/origin/HEAD issue = repo.git( diff --git a/gitlab_sync/repository.py b/gitlab_sync/repository.py index b66b4ff..780083c 100644 --- a/gitlab_sync/repository.py +++ b/gitlab_sync/repository.py @@ -1,24 +1,39 @@ """Module for the collection and representation of information on repositories. """ +import abc import asyncio import os import pathlib import subprocess -import attr import typing import aiohttp +import attr import gitlab_sync _DEV_NULL = open(os.devnull, "r+b") +@attr.s(auto_attribs=True) +class RepoInterface(abc.ABC): + repo_path: pathlib.Path + + @abc.abstractmethod + def create(self): + pass + + @abc.abstractmethod + def delete(self): + pass + + @attr.s(auto_attribs=True) class Repository: base_path: pathlib.Path gitlab_path: pathlib.Path id: typing.Optional[int] = None + last_gitlab_path: typing.Optional[str] = None @property def local_path(self): @@ -53,6 +68,7 @@ def enumerate_local(base_path): del dirs[:] gitlab_path = pathlib.Path(root).relative_to(base_path) repo = Repository(base_path, gitlab_path) + # remote project id result = repo.git( "config", "--local", @@ -62,6 +78,17 @@ def enumerate_local(base_path): ) if not result.returncode: repo.id = int(result.stdout.rstrip()) + # remote gitlab path + result = repo.git( + "config", + "--local", + "gitlab-sync.project-path", + stdout=subprocess.PIPE, + universal_newlines=True, + ) + if not result.returncode: + repo.last_gitlab_path = result.stdout.rstrip() + yield repo diff --git a/gitlab_sync/strategy.py b/gitlab_sync/strategy.py index 344b666..c5efd42 100644 --- a/gitlab_sync/strategy.py +++ b/gitlab_sync/strategy.py @@ -6,48 +6,130 @@ """ import shutil +import typing import gitlab_sync.operations import gitlab_sync.repository from gitlab_sync import logger +from gitlab_sync.repository import Repository -# XXX: it may be good to generate the maps in a helper method -def mirror(config): - """Perform necissary actions to update a local copy using backup logic.""" +def _get_repo_maps(config): locals_ = list(gitlab_sync.repository.enumerate_local(config.base_path)) - # TODO: update paths to be namespaces in other places remotes = list( gitlab_sync.repository.enumerate_remote(config) ) + logger.debug("local repos found: %r", locals_) + logger.debug("remote repos found: %r", remotes) + return _classify_repos(locals_, remotes) + + +def _classify_repos(locals_: typing.List[Repository], remotes: typing.List[Repository]): + """Return a plan for operations to synchronise remote and local repositories. - remoteless = [repo for repo in locals_ if repo.id is None] - if remoteless: - # TODO: subclass exceptions - # low chance of being due to a failure between git-init and git-config - raise Exception("Unexpected directories.") + The granularity of the plan lets strategies pick and mix what they do, and + detect situations that are erronious for the chosen strategy. + + The intention of the generic plan is to make the local and remote repos + equal in terms of relative location, and git trees. + + update_local + update_remote can result in conflicts that need resolving, + some strategies (e.g. mirror) do not expect this to happen so raise if an + issue is found so the user can deal with it. It would be nice if the classification + happened here following a fetch, then this plan knows everything. The git sync + makes testing require some mocking. + + """ local_map = {repo.id: repo for repo in locals_} remote_map = {repo.id: repo for repo in remotes} - logger.debug("local repos found: %r", locals_) - logger.debug("remote repos found: %r", remotes) - delete_map = {} + move_resolve = [] + for id_, local in local_map.items(): + remote = remote_map[id_] + if (local.last_gitlab_path and local.last_gitlab_path != local.gitlab_path and + local.gitlan_path != remote.gitlab_path): + move_resolve.append((local, remote)) + + delete_local = [] for id_ in local_map.keys() - remote_map.keys(): repo = local_map.pop(id_) - delete_map[repo.id] = repo + delete_local.append((repo,)) - create_map = {} + create_local = [] for id_ in remote_map.keys() - local_map.keys(): repo = remote_map.pop(id_) - create_map[repo.id] = repo + create_local.append((repo,)) - move_map = {} + # FIXME: move operation updates project-path/last_gitlab_path + # FIXME: clone operation sets project-path/last_gitlab_path + # when there's a remove move, the remote has to be updated, so + # can't update a repo that needs to be resolved. + move_local = [] for id_, local in local_map.items(): remote = remote_map[id_] - if remote.gitlab_path != local.gitlab_path: - move_map[id_] = (remote, local.gitlab_path, remote.gitlab_path) + # repo hasn't moved locally but has remotely + if (local.gitlab_path == local.last_gitlab_path and + remote.gitlab_path != local.gitlab_path): + move_local.append((repo, remote.gitlab_path)) + # what if both have changed? conflict... separate resolve operation + # how can you tell a local move from a remote move? store local rel path + # in local copy, if repo has moved removely then at locally = local rel + # this really returns a "plan", but the strategies select a subset of it + # and enforce their own constraints + # ({operation: [(repo, *args), ...]}) + """ + [ + (create_local, []), + (move_local, []), + (update_local, []), + (delete_local, []), + (create_remote, []), + (move_remote, []), + (update_remote, []), + (delete_remote, []), + + (move_resolve, []), + ] + """ + update_local = [ + (repo, ) for repo in remote_map.values() + ] + + create_remote = [(repo, ) for repo in locals_ if repo.id is None] + + move_remote = [] + for id_, local in local_map.items(): + remote = remote_map[id_] + # TODO: guard heavily against moving things around by mistake due to code errors + # make sure there is good test coverage + if (local.last_gitlab_path and + local.last_gitlab_path != local.gitlab_path and + remote.gitlab_path != local.gitlab_path): + move_local.append((repo, remote.gitlab_path)) + + # TODO: see about modification times + update_remote = [] + delete_remote = [] + + return [ + # maybe update should be a single final operation, but there are levels of update + ("create_local", create_local), + ("move_local", move_local), + ("update_local", update_local), + ("delete_local", delete_local), + ("create_remote", create_remote), + ("move_remote", move_remote), + ("update_remote", update_remote), + ("delete_remote", delete_remote), + ("move_resolve", move_resolve), + ] + + +# XXX: it may be good to generate the maps in a helper method +def mirror(config): + """Perform necissary actions to update a local copy using backup logic.""" + # raise if any create_remote, move_remote, delete_remote (do not run update_remote) - update_map = remote_map for repo in sorted(delete_map.values()): logger.info("deleting %s", repo) diff --git a/tests/test_strategy.py b/tests/test_strategy.py new file mode 100644 index 0000000..4be30bb --- /dev/null +++ b/tests/test_strategy.py @@ -0,0 +1,25 @@ +"""Module for the testing the internals of the strategy module.""" +""" +class Sync: + local = None + remote = None + +class Remote: + exists = False + + def create(): pass + def delete(): pass + +class Local: + exists = False + + def create(): pass + def delete(): pass + +""" + + +def test_full_sync_plan(): + # TODO: make input which has sync-couples for: + # {create,move,update,delete} {local,remote} + move resolve + pass