From 0ae77b8e74bdad28b1dbd65c3b7127cdd891a568 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Tue, 18 Dec 2018 21:26:35 -0600 Subject: [PATCH 1/2] WIP: golang --- README.md | 9 +- cmd/gitlab-sync/main.go | 53 ++++++++ config.go | 189 ++++++++++++++++++++++++++ gitlab_sync/__init__.py | 6 - gitlab_sync/cli.py | 49 ------- gitlab_sync/config.py | 124 ----------------- gitlab_sync/operations.py | 92 ------------- gitlab_sync/repository.py | 279 -------------------------------------- gitlab_sync/strategy.py | 76 ----------- glide.lock | 98 +++++++++++++ glide.yaml | 16 +++ go.mod | 13 ++ go.sum | 29 ++++ repository.go | 122 +++++++++++++++++ service.go | 123 +++++++++++++++++ service_test.go | 10 ++ setup.cfg | 4 - setup.py | 27 ---- strategy.go | 124 +++++++++++++++++ strategy_test.go | 83 ++++++++++++ 20 files changed, 868 insertions(+), 658 deletions(-) create mode 100644 cmd/gitlab-sync/main.go create mode 100644 config.go delete mode 100644 gitlab_sync/__init__.py delete mode 100644 gitlab_sync/cli.py delete mode 100644 gitlab_sync/config.py delete mode 100644 gitlab_sync/operations.py delete mode 100644 gitlab_sync/repository.py delete mode 100644 gitlab_sync/strategy.py create mode 100644 glide.lock create mode 100644 glide.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 repository.go create mode 100644 service.go create mode 100644 service_test.go delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 strategy.go create mode 100644 strategy_test.go diff --git a/README.md b/README.md index af6bedf..00714dc 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,13 @@ The local copies should not be modified by users. ## To do + * separate out packages so it's clear where things are coming from + * use Gitlab/Service interface to get remote repos and classify for ops + * look up reccomended repo structure, maybe move code into pkg/ * flesh out integration tests * cater for new repositories being made locally and pushed remotely - * compare (toasted-)marshmallow as a replacement for attr+voluptious - marshmallow uses attr + * document/give example of extending for a new service provider + +## Extension + +Consider other service providers such as GitHub, gogs, then call just git-sync and propose for git packages? diff --git a/cmd/gitlab-sync/main.go b/cmd/gitlab-sync/main.go new file mode 100644 index 0000000..2157143 --- /dev/null +++ b/cmd/gitlab-sync/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + // "github.com/spf13/cobra" + "github.com/Code0x58/gitlab-sync" + "sync" +) + +func main() { + var c, err = gitlabsync.LoadConfig() + if err != nil { + panic(fmt.Errorf("Fatal error in config: %s\n", err)) + } + + // run a goroutine worker pool + syncs := make(chan gitlabsync.SyncRun, c.MaxParallelSyncs) + var wg sync.WaitGroup + for i := uint(0); i < c.MaxParallelSyncs; i++ { + wg.Add(1) + go func() { + defer wg.Done() + sync := <-syncs + fmt.Printf("%#v\n", sync.pathConfig) + fmt.Printf("access-token: %s\n", sync.pathConfig.AccessToken) + // gitlabsync.LocalRepos(&pathConfig) + + var remoteRepos []*remoteRepo + { + service, err := NewGitlab(s.userConfig, s.pathConfig) + if err != nil { + panic(err) + } + fmt.Print("About to get repos\n") + remoteRepos, err = service.RemoteRepos() + if err != nil { + panic(err) + } + for _, remote := range remotes { + fmt.Printf("%#v\n", *remote) + } + } + gitlabsync.newMetaSyncPlan(locals, remotes) + }() + } + // feed the worker pool + for id, pathConfig := range c.PathConfigs { + syncs <- gitlabsync.SyncRun{id, c, &pathConfig} + } + // starve and wait for the pool to finish + close(syncs) + wg.Wait() +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..f9900d8 --- /dev/null +++ b/config.go @@ -0,0 +1,189 @@ +package gitlabsync + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strings" + + homedir "github.com/mitchellh/go-homedir" + + "github.com/BurntSushi/toml" +) + +type UserConfig struct { + // naming of the last two should be improvable + PathConfigs []PathConfig `toml:"path-config"` + MaxParallelSyncs uint `toml:"max-parallel-syncs"` + MaxParallelSyncOperations uint `toml:"max-parallel-sync-operationns"` +} + +type PathConfig struct { + BasePath string + AccessToken string + Strategy string + APIURL string + GitURL string + StripPath bool + RemotePaths []string +} + +func (c *PathConfig) UnmarshalTOML(data interface{}) error { + d, _ := data.(map[string]interface{}) + { // required string/[]string -> string + typeError := errors.New("path-configs.access-token must be a string or list of strings") + i, exists := d["access-token"] + if !exists { + return errors.New("path-configs.access-token is required") + } + switch v := i.(type) { + case string: + c.AccessToken = v + case []interface{}: + if len(v) == 0 { + return typeError + } + a := make([]string, len(v)) + for j, p := range v { + s, ok := p.(string) + if !ok { + return typeError + } + a[j] = s + } + command := exec.Command(a[0], a[1:]...) + out, err := command.Output() + if err != nil { + return fmt.Errorf("Getting the acess token failed with %q failed: %s", a, err) + } + c.AccessToken = strings.TrimSuffix(string(out), "\n") + default: + return fmt.Errorf("Bad %#v", v) //typeError + } + } + { // required string + i, exists := d["base-path"] + if !exists { + return errors.New("path-configs.base-path is required") + } + v, ok := i.(string) + if !ok { + return errors.New("path-configs.base-path must be a string") + } + var err error + c.BasePath, err = homedir.Expand(v) + if err != nil { + return err + } + } + { // required string + i, exists := d["strategy"] + if !exists { + return errors.New("path-configs.strategy is required") + } + v, ok := i.(string) + if !ok { + return errors.New("path-configs.strategy must be a string") + } + c.Strategy = v + } + { // optional string + i, exists := d["http-url"] + if exists { + v, ok := i.(string) + if !ok { + return errors.New("path-configs.api-url must be a string") + } + c.APIURL = v + } else { + c.APIURL = "https://gitlab.com/" + } + } + { // optional string + i, exists := d["git-url"] + if exists { + v, ok := i.(string) + if !ok { + return errors.New("path-configs.git-url must be a string") + } + c.GitURL = v + } else { + c.GitURL = "git+ssh://git@gitlab.com/" + } + } + { // optional bool + i, exists := d["strip-path"] + if exists { + v, ok := i.(bool) + if !ok { + return errors.New("path-configs.strip-path must be a boolean") + } + c.StripPath = v + } else { + c.StripPath = false + } + } + { // optional []string + typeError := errors.New("path-configs.paths must be a list of strings") + i, exists := d["paths"] + if exists { + v, ok := i.([]interface{}) + if !ok { + return typeError + } + a := make([]string, len(v)) + for j, p := range v { + s, ok := p.(string) + if !ok { + return typeError + } + a[j] = s + } + c.RemotePaths = a + } else { + c.RemotePaths = []string{} + } + } + return nil +} + +// FIXME: separate file finding and loading, use ioutil to read bytes from file +// then have this method use that, then defaults can be tested +func LoadConfig() (*UserConfig, error) { + // TODO: put the defaults in the userconfig instantiation - works for top level but not for lists/maps... have to use primitives/decode one at a time? + var c *UserConfig + // load in the base config (before validation+transforms) + for _, p := range []string{ + "${GITLAB_SYNC_CONFIG}", + "~/.config/gitlab-sync.toml", + "~/.gitlab-sync.toml", + } { + var err error + p, err = homedir.Expand(p) + if err != nil { + return nil, err + } + p := os.ExpandEnv(p) + config := UserConfig{ + MaxParallelSyncs: 1, + MaxParallelSyncOperations: 1, + } + + _, err = toml.DecodeFile(p, &config) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("Unable load %s: %s", p, err) + } + + c = &config + break + } + if c == nil { + return nil, errors.New("No config file found") + } + + return c, nil +} diff --git a/gitlab_sync/__init__.py b/gitlab_sync/__init__.py deleted file mode 100644 index 7e34a51..0000000 --- a/gitlab_sync/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -import logging -logger = logging.getLogger("gitlab-sync") - - -class ConfigurationError(ValueError): - """Raised for errors during loading config.""" diff --git a/gitlab_sync/cli.py b/gitlab_sync/cli.py deleted file mode 100644 index c25020f..0000000 --- a/gitlab_sync/cli.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import logging - -import click -import gitlab_sync -import gitlab_sync.strategy -from gitlab_sync.config import find_and_load_config -from gitlab_sync import ConfigurationError, logger - - -@click.group() -@click.option("-v", "--verbose", count=True) -@click.pass_context -def main(ctx, verbose): - log_level = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG][ - min(verbose, 3) - ] - logging.basicConfig( - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S%z", - level=log_level, - ) - gitlab_sync.tee_git = log_level == logging.DEBUG - - try: - config = find_and_load_config() - except ConfigurationError as e: - logger.error(str(e)) - raise SystemExit(1) - - ctx.obj = config - - -@main.command("local-update", short_help="synchronise managed repositories") -@click.pass_context -def local_update(ctx): - """Manage local copies of repositories on GitLab.""" - run_configs = ctx.obj - # XXX: more like mirror really, and that should be a config only thing, - # not something you choose on a run by run basis… - # XXX: could return projectless repos (new) and missing repos? maybe - # guard against deleting all projects if restoring config from backup - # but not repo directory - for config in run_configs.values(): - config.strategy(config) - - -if __name__ == "__main__": - main() diff --git a/gitlab_sync/config.py b/gitlab_sync/config.py deleted file mode 100644 index cb6ccaa..0000000 --- a/gitlab_sync/config.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Module for the config schema, loading, and runtime representation. - -The only method expected to be used from this module is find_and_load_config, -other members should be considered private. - -""" -import os -import subprocess -import sys -import types -import typing -from pathlib import Path - -import attr -import gitlab_sync.strategy -import toml -from gitlab_sync import ConfigurationError -from voluptuous import All, And, Any, Boolean, Invalid, MultipleInvalid, Optional, Replace, Required, Schema, Url - - -def absolute_dir_path(string) -> Path: - """Make sure the input path string is absolute.""" - path = Path(string) - if not path.is_absolute(): - path = path.expanduser() - if not path.is_absolute(): - raise Invalid("path must be absolute (~ allowed)") - path.mkdir(parents=True, exist_ok=True) - return path - - -def gitlab_path(string) -> Path: - return Path(string) - - -def string_or_source(value: typing.Union[str, typing.List[str]]) -> str: - """Requires a literal string, or list of strings describing a command.""" - if isinstance(value, str): - return value - return subprocess.run( - value, - stderr=sys.stderr, - stdout=subprocess.PIPE, - universal_newlines=True, - check=True, - ).stdout.strip() - - -def valid_strategy(value: str) -> typing.Callable[["RunConfig"], None]: - """Lookup a strategy given it's name.""" - strategy = getattr(gitlab_sync.strategy, value, None) - if not isinstance(strategy, types.FunctionType): - raise Invalid("Must be the name of a strategy.") - return strategy - - -def strip_path_single_path(copy_config): - if copy_config.get("strip-path") and len(copy_config["paths"]) != 1: - raise Invalid("strip-path can only be used when paths has one element") - return copy_config - - -schema = Schema({ - Required(absolute_dir_path): All({ - Required(All("access-token", Replace("-", "_"))): And( - Any( - str, - All([str]), - ), string_or_source, - ), - Required("paths"): [gitlab_path], - Required("strategy"): valid_strategy, - Optional(All("gitlab-http", Replace("-", "_"))): Url(), - Optional(All("gitlab-git", Replace("-", "_"))): Url(), - Optional(All("strip-path", Replace("-", "_"))): Boolean, - }, strip_path_single_path), -}) - - -def find_config() -> Path: - """Find a config file to use, or raise if none available.""" - environ = "GITLAB_SYNC_CONFIG" - path = os.environ.get(environ) - if path: - path = Path(path) - if path.is_file(): - return path - else: - raise ConfigurationError("{} given in {} is not a file".format(path, environ)) - home = Path.home() - for path in (home / ".config/gitlab-sync.toml", home / ".gitlab-sync.toml"): - if path.is_file(): - return path - raise ConfigurationError("No config file found") - - -def load_config(file_: typing.TextIO) -> dict: - """Load and validate config from a file.""" - try: - return schema(toml.load(file_)) - except (toml.TomlDecodeError, IOError, FileNotFoundError, TypeError) as e: - raise ConfigurationError("Unable to load config: %s" % e) from e - except MultipleInvalid as e: - raise ConfigurationError("Config not valid: %s" % e) from e - - -@attr.s(auto_attribs=True) -class RunConfig: - base_path: Path - paths: typing.List[Path] - access_token: str - strategy: typing.Callable[["RunConfig"], None] - gitlab_http: str = "https://gitlab.com/" - gitlab_git: str = "git+ssh://git@gitlab.com/" - strip_path: bool = False - - -def find_and_load_config() -> typing.List[RunConfig]: - """Top level method to acquire config for gitlab-sync.""" - data = load_config(find_config().open()) - return { - path: RunConfig(path, **settings) - for path, settings in data.items() - } diff --git a/gitlab_sync/operations.py b/gitlab_sync/operations.py deleted file mode 100644 index d0461d0..0000000 --- a/gitlab_sync/operations.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Module containing low level operations on repositories. - -These operation don't work out what to do at any point, that logic is left to -the strategy module. - -""" -import os -import shutil -import subprocess - -from gitlab_sync import logger - - -def clone(config, local, remote): - """Clone a new repository.""" - os.makedirs(str(local.absolute_path)) - local.git("init", ".") - local.gitlab_project_id = remote.gitlab_project_id - local.gitlab_path = remote.gitlab_path - local.git( - "remote", "add", "origin", - config.gitlab_git + "%s.git" % remote.gitlab_path, - ) - update_local(local) - - -def update_local(local): - """Update master from the remote.""" - local.git("fetch") - # get refs/remotes/origin/HEAD - result = local.git( - "remote", - "set-head", - "origin", - "--auto", - check=False, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - issue = result.stderr.rstrip() - if not result.returncode: - # read which branch the remote HEAD points to - remote_head = local.git( - "symbolic-ref", - "refs/remotes/origin/HEAD", - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout.rstrip() - # checkout and track the branch that the remote HEAD points to - result = local.git( - "checkout", - "--track", - remote_head, - check=False, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - if result.returncode: - issue = result.stderr - # if the branch is already tracked, then just check out the tip of it - name = remote_head.rpartition("/")[2] - if issue.startswith(r"fatal: A branch named '%s' already exists." % name): - local.git("reset", "--hard", remote_head) - else: - raise Exception(issue.rstrip()) - else: - logger.debug("`git checkout --track %s` worked", remote_head) - elif issue.endswith("error: Cannot determine remote HEAD"): - logger.debug("%s is an empty project", local) - else: - raise Exception(issue) - # mirror only logic - local.git("clean", "-d", "--force") - - -def delete_local(repo): - logger.debug("removing %s", repo) - shutil.rmtree(str(repo.absolute_path)) - prune = repo.absolute_path.parent - while prune != repo.base_path: - try: - os.rmdir(str(prune)) - except OSError: - # assuming this is because the directory isn't empty - break - logger.debug("pruned %s", prune) - - -def clean(repo): - logger.debug("cleaning %s", repo) - repo.git("remote", "prune", "origin") - repo.git("gc", "--auto") diff --git a/gitlab_sync/repository.py b/gitlab_sync/repository.py deleted file mode 100644 index 1ff8764..0000000 --- a/gitlab_sync/repository.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Module for the collection and representation of information on repositories. - -""" -import asyncio -import os -import pathlib -import subprocess -import attr -import typing - -import aiohttp -import gitlab_sync - -_DEV_NULL = open(os.devnull, "r+b") - - -@attr.s(auto_attribs=True) -class LocalRepository: - base_path: pathlib.Path - relative_path: pathlib.Path - - @property - def absolute_path(self): - return self.base_path / self.relative_path - - def git(self, *git_args, **run_kwargs): - """Run a command in git using `subprocess.run` where `check=True` by default.""" - run_kwargs.setdefault("check", True) - if not gitlab_sync.tee_git: - run_kwargs.setdefault("stdout", _DEV_NULL) - run_kwargs.setdefault("stderr", _DEV_NULL) - command = ["git", "-C", str(self.absolute_path)] + list(git_args) - return subprocess.run(command, **run_kwargs) - - def _get_gitlab_project_id(self): - if not hasattr(self, "_gitlab_project_id"): - result = self.git( - "config", - "--local", - "gitlab-sync.project-id", - stdout=subprocess.PIPE, - universal_newlines=True, - check=False, - ) - if result.returncode: - self._gitlab_project_id = None - else: - self._gitlab_project_id = int(result.stdout.rstrip()) - return self._gitlab_project_id - - def _set_gitlab_project_id(self, value): - self.git( - "config", - "--local", - "gitlab-sync.project-id", - str(value), - ) - self._gitlab_project_id = value - - def _get_gitlab_path(self): - if not hasattr(self, "_gitlab_path"): - result = self.git( - "config", - "--local", - "gitlab-sync.gitlab-path", - stdout=subprocess.PIPE, - universal_newlines=True, - check=False, - ) - if result.returncode: - self._gitlab_path = None - else: - self._gitlab_path = pathlib.Path(result.stdout.rstrip()) - return self._gitlab_path - - def _set_gitlab_path(self, value): - self.git( - "config", - "--local", - "gitlab-sync.gitlab-path", - str(value), - ) - self._gitlab_path = value - - gitlab_project_id = property(_get_gitlab_project_id, _set_gitlab_project_id) - gitlab_path = property(_get_gitlab_path, _set_gitlab_path) - - def __str__(self): - return str(self.gitlab_path) - - def __gt__(self, other): - return self.absolute_path > other.absolute_path - - @classmethod - def from_remote(cls, config, remote): - """Return an instance suitable for cloning into.""" - if config.strip_path: - relative_path = remote.gitlab_path.relative_to(config.paths[0]) - else: - relative_path = remote.gitlab_path - instance = cls(config.base_path, relative_path) - instance._gitlab_path = remote.gitlab_path - return instance - - -@attr.s(auto_attribs=True) -class GitlabRepository: - gitlab_path: pathlib.Path - gitlab_project_id: typing.Optional[int] = None - - def __str__(self): - return str(self.gitlab_path) - - def __gt__(self, other): - return self.gitlab_path > other.gitlab_path - - -@attr.s(auto_attribs=True) -class RemoteRepository: - relative_path: pathlib.Path - absolute_path: pathlib.Path - gitlab_project_id: typing.Optional[int] = None - - -def enumerate_local(base_path): - """Return all local repositories under a given path.""" - for root, dirs, files in os.walk(base_path): - if ".git" not in dirs: - continue - del dirs[:] - store_path = pathlib.Path(root).relative_to(base_path) - repo = LocalRepository(base_path, store_path) - yield repo - - -class NotAGroup(Exception): - pass - - -class ProjectCollector(object): - """ - Class to collect Repositories from GitLab using asynchronous HTTP - requests to speed up traversing tree structures. - - """ - - def __init__(self, config): - self.config = config - - def filter_projects(self, projects): - """Yield repository objects for projects of interest.""" - for project in projects: - path = pathlib.Path(project["path_with_namespace"]).parts - for filter_path in self.config.paths: - filter_parts = filter_path.parts - if path[:len(filter_parts)] == filter_parts: - yield GitlabRepository(pathlib.Path(project["path_with_namespace"]), project["id"]) - break - else: - gitlab_sync.logger.debug( - "Skipping %s as it does not match a filter path", - project["path_with_namespace"], - ) - - async def _get_user_projects(self, user): - projects = [] - async with self.session.get( - "{}api/v4/users/{}/projects".format(self.config.gitlab_http, user), - params={"per_page": 100, "page": 1, "simple": 1}, - ) as response: - projects.extend(await response.json()) - next_page = response.headers.get("X-Next-Page") - - while next_page: - async with self.session.get( - "{}api/v4/users/{}/projects".format(self.config.gitlab_http, user), - params={"per_page": 100, "page": next_page, "simple": 1}, - ) as response: - projects.extend(await response.json()) - next_page = response.headers.get("X-Next-Page") - - return self.filter_projects(projects) - - async def _get_group_projects(self, group): - projects = [] - async with self.session.get( - "{}api/v4/groups/{}/projects".format(self.config.gitlab_http, group), - params={"per_page": 100, "page": 1, "simple": 1}, - ) as response: - data = await response.json() - projects.extend(data) - next_page = response.headers.get("X-Next-Page") - - while next_page: - async with self.session.get( - "{}api/v4/groups/{}/projects".format(self.config.gitlab_http, group), - params={"per_page": 100, "page": next_page, "simple": 1}, - ) as response: - projects.extend(await response.json()) - next_page = response.headers.get("X-Next-Page") - - return self.filter_projects(projects) - - async def _get_group_subgroups(self, group): - """Yields a (sub)group names/ids""" - groups = [] - async with self.session.get( - "{}api/v4/groups/{}/subgroups".format(self.config.gitlab_http, group), - params={"per_page": 100, "page": 1}, - ) as response: - data = await response.json() - if not isinstance(data, list): - raise NotAGroup() - groups.extend(data) - next_page = response.headers.get("X-Next-Page") - # yield the given group when we know it isn't a user - yield group - - while next_page: - async with self.session.get( - "{}api/v4/groups/{}/subgroups".format(self.config.gitlab_http, group), - params={"per_page": 100, "page": next_page}, - ) as response: - data = await response.json() - groups.extend(data) - next_page = response.headers.get("X-Next-Page") - - for group_data in groups: - yield group_data["id"] - async for subgroup_id in self._get_group_subgroups(group_data["id"]): - yield subgroup_id - - async def _get_entity_projects(self, entity): - try: - projects = [] - for projects_future in asyncio.as_completed( - [ - asyncio.ensure_future(self._get_group_projects(group)) async - for group in self._get_group_subgroups(entity) - ] - ): - projects.extend(await projects_future) - return projects - except NotAGroup: - pass - return await self._get_user_projects(entity) - - async def _get_paths(self, paths): - entities = {path.parts[0] for path in paths} - all_paths = [] - async with aiohttp.ClientSession( - headers={"Private-Token": self.config.access_token} - ) as self.session: - paths_futures = [] - for paths_future in asyncio.as_completed( - [ - asyncio.ensure_future(self._get_entity_projects(entity)) - for entity in entities - ] - ): - paths_futures.append(paths_future) - for paths in asyncio.as_completed(paths_futures): - all_paths.extend(await paths) - return all_paths - - def collect_paths(self): - """ - Return a list of Repository object for projects under the given paths in GitLab. - - """ - loop = asyncio.get_event_loop() - return loop.run_until_complete(self._get_paths(self.config.paths)) - - -def enumerate_remote(config): - """Return all repositories available to the given access token.""" - # TODO: return a generator which collects from asyncio - # TODO: think how this can work where users want to clone everything under their user/group - return ProjectCollector(config).collect_paths() diff --git a/gitlab_sync/strategy.py b/gitlab_sync/strategy.py deleted file mode 100644 index d1bc3af..0000000 --- a/gitlab_sync/strategy.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Module for top level strategies for local copies. - -The methods in here direct lower level operations. The idea is to have as -little if/else handling as possible, with lower level methods having 0 -knowledge of their use. - -""" -import shutil - -import gitlab_sync.operations -import gitlab_sync.repository -from gitlab_sync import logger - - -# XXX: it may be good to generate the maps in a helper method -def mirror(config): - """Perform necissary actions to update a local copy using backup logic.""" - locals_ = list(gitlab_sync.repository.enumerate_local(config.base_path)) - # TODO: update paths to be namespaces in other places - remotes = list( - gitlab_sync.repository.enumerate_remote(config) - ) - - remoteless = [repo for repo in locals_ if repo.gitlab_project_id is None] - if remoteless: - # TODO: subclass exceptions - # low chance of being due to a failure between git-init and git-config - raise Exception("Unexpected directories.") - local_map = {repo.gitlab_project_id: repo for repo in locals_} - remote_map = {repo.gitlab_project_id: repo for repo in remotes} - logger.debug("local repos found: %r", locals_) - logger.debug("remote repos found: %r", remotes) - - delete_map = {} - for id_ in local_map.keys() - remote_map.keys(): - repo = local_map.pop(id_) - delete_map[repo.gitlab_project_id] = repo - - create_map = {} - for id_ in remote_map.keys() - local_map.keys(): - repo = remote_map.pop(id_) - create_map[repo.gitlab_project_id] = repo - - move_map = {} - for id_, local in local_map.items(): - remote = remote_map[id_] - if local.gitlab_path and remote.gitlab_path != local.gitlab_path: - move_map[id_] = (remote, local.gitlab_path, remote.gitlab_path) - - update_map = { - id_: gitlab_sync.repository.LocalRepository.from_remote(config, remote) - for id_, remote in remote_map.items() - if id_ not in create_map - } - - for repo in sorted(delete_map.values()): - logger.info("deleting %s", repo) - gitlab_sync.operations.delete_local(repo) - # TODO: think about being definsive against errors reading from GitLab - # maybe GitLab retains projects in the database after they are deleted? - # tombstones would be nice - - for repo, old_gitlab_path, new_gitlab_path in sorted(move_map.values()): - logger.info("moving %s to %s", old_gitlab_path, new_gitlab_path) - shutil.move(str(config.base_path / old_gitlab_path), str(config.base_path / new_gitlab_path)) - - for repo in sorted(update_map.values()): - logger.info("updating %s", repo) - gitlab_sync.operations.update_local(repo) - logger.info("cleaning %s", repo) - gitlab_sync.operations.clean(repo) - - for remote in sorted(create_map.values()): - logger.info("copying %s", remote) - local = gitlab_sync.repository.LocalRepository.from_remote(config, remote) - gitlab_sync.operations.clone(config, local, remote) diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..b1af43d --- /dev/null +++ b/glide.lock @@ -0,0 +1,98 @@ +hash: 100a36f6fc55de441d11a7db24d596c95d4e164bbb1b590acb83af6ca647acd4 +updated: 2018-12-26T08:24:14.237127803Z +imports: +- name: github.com/BurntSushi/toml + version: 3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005 +- name: github.com/fsnotify/fsnotify + version: ccc981bf80385c528a65fbfdd49bf2d8da22aa23 +- name: github.com/go-playground/mold + version: 6bc20ce40733e8e04c2fda4523a957dd8e198948 +- name: github.com/go-playground/validator + version: 0277b12d53df79c9dbf7311cb07fa9c81ed621bb +- name: github.com/golang/protobuf + version: 1d3f30b51784bec5aad268e59fd3c2fc1c2fe73f + subpackages: + - proto +- name: github.com/google/go-querystring + version: 44c6ddd0a2342c386950e880b658017258da92fc + subpackages: + - query +- name: github.com/hashicorp/hcl + version: 65a6292f0157eff210d03ed1bf6c59b190b8b906 + subpackages: + - hcl/ast + - hcl/parser + - hcl/printer + - hcl/scanner + - hcl/strconv + - hcl/token + - json/parser + - json/scanner + - json/token +- name: github.com/magiconair/properties + version: 7c38529aac7222391b7a2661365177a97e21b998 +- name: github.com/mitchellh/go-homedir + version: ae18d6b8b3205b561c79e8e5f69bff09736185f4 +- name: github.com/mitchellh/mapstructure + version: 3536a929edddb9a5b34bd6861dc4a9647cb459fe +- name: github.com/pelletier/go-toml + version: 27c6b39a135b7dc87a14afb068809132fb7a9a8f +- name: github.com/spf13/afero + version: a5d6946387efe7d64d09dcba68cdd523dc1273a3 + subpackages: + - mem +- name: github.com/spf13/cast + version: 8c9545af88b134710ab1cd196795e7f2388358d7 +- name: github.com/spf13/cobra + version: ef82de70bb3f60c65fb8eebacbb2d122ef517385 +- name: github.com/spf13/jwalterweatherman + version: 94f6ae3ed3bceceafa716478c5fbf8d29ca601a1 +- name: github.com/spf13/pflag + version: 24fa6976df40757dce6aea913e7b81ade90530e1 +- name: github.com/spf13/viper + version: 6d33b5a963d922d182c91e8a1c88d81fd150cfd4 +- name: github.com/stretchr/testify + version: f35b8ab0b5a2cef36673838d662e249dd9c94686 + subpackages: + - assert +- name: github.com/xanzy/go-gitlab + version: 6cd773821c396eeffbad661e28e9d2b2a06d9611 +- name: golang.org/x/net + version: 927f97764cc334a6575f4b7a1584a147864d5723 + subpackages: + - context + - context/ctxhttp +- name: golang.org/x/oauth2 + version: d668ce993890a79bda886613ee587a69dd5da7a6 + subpackages: + - internal +- name: golang.org/x/sys + version: b4a75ba826a64a70990f11a225237acd6ef35c9f + subpackages: + - unix +- name: golang.org/x/text + version: 17bcc049122f272a32787ba38073ee47433023e9 + subpackages: + - transform + - unicode/norm +- name: google.golang.org/appengine + version: e9657d882bb81064595ca3b56cbe2546bbabf7b1 + subpackages: + - internal + - internal/base + - internal/datastore + - internal/log + - internal/remote_api + - internal/urlfetch + - urlfetch +- name: gopkg.in/yaml.v2 + version: 51d6538a90f86fe93ac480b35f37b2be17fef232 +testImports: +- name: github.com/davecgh/go-spew + version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d + subpackages: + - spew +- name: github.com/pmezard/go-difflib + version: 792786c7400a136282c1664665ae0a8db921c6c2 + subpackages: + - difflib diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..decd1c7 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,16 @@ +package: github.com/Code0x58/gitlab-sync +import: +- package: github.com/spf13/cobra + version: ^0.0.3 +- package: github.com/xanzy/go-gitlab + version: ^0.12.0 +- package: github.com/stretchr/testify + version: ^1.2.2 + subpackages: + - assert +- package: github.com/go-playground/mold + version: ^2.2.0 +- package: github.com/go-playground/validator + version: ^9.24.0 +- package: github.com/BurntSushi/toml + version: ^0.3.1 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..aaf5150 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/Code0x58/gitlab-sync + +require ( + github.com/BurntSushi/toml v0.3.1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mitchellh/go-homedir v1.0.0 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.2.2 + github.com/xanzy/go-gitlab v0.12.0 + golang.org/x/net v0.0.0-20181220203305-927f97764cc3 // indirect + golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect + google.golang.org/appengine v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..eafe1c1 --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/xanzy/go-gitlab v0.12.0 h1:rs40DfrKvJoIluarQJcFmOADVMlgFGDMXvnEeQpjqGg= +github.com/xanzy/go-gitlab v0.12.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/repository.go b/repository.go new file mode 100644 index 0000000..12bb565 --- /dev/null +++ b/repository.go @@ -0,0 +1,122 @@ +package gitlabsync + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +type localRepo struct { + BasePath string // base path on local system + CurrentRelativePath string // current relative path on local system + storedRelativePath string // previous relative path on local system + RemoteId string // id of project in the remote service +} + +type remoteRepo struct { + Path string // path of the repo in service + Id string // id of project in service +} + +// Run a git command on the local repository +func (l *localRepo) git(args ...string) *exec.Cmd { + path := filepath.Join(l.BasePath, l.CurrentRelativePath) + all_args := make([]string, len(args)+3) + all_args[0] = "git" + all_args[1] = "-C" + all_args[2] = path + copy(all_args[3:], args) + return exec.Command("git", all_args...) +} + +// Get a config item from the local repository, or an empty string +func (l *localRepo) config(name string) string { + item := fmt.Sprintf("gitlab-sync.%s", name) + out, _ := l.git("config", "--local", item).Output() + return string(out) +} + +func (l *localRepo) setConfig(name string, value string) error { + item := fmt.Sprintf("gitlab-sync.%s", name) + _, err := l.git("config", "--local", item, value).Output() + // TODO(obristow): think about a panic in this situation + return err +} + +// Return the relative path that was last seen +func (l *localRepo) StoredRelativePath() string { + if len(l.storedRelativePath) == 0 { + l.storedRelativePath = l.config("relative-path") + } + return l.storedRelativePath +} + +// create the local repository +func (l *localRepo) Create() error { + _, err := l.git("init", ".").Output() + if err != nil { + return err + } + l.setConfig("relative-path", l.CurrentRelativePath) + l.setConfig("remote-id", l.RemoteId) + return nil +} + +// rename the local repository +func (l *localRepo) Rename(relativePath string) error { + // TODO(obristow): move to the given relative path + /* + out, err := l.git("init", ".").Output() + l.CurrentRelativePath = relativePath + l.setConfig("relative-path", relativePath) + */ + return nil +} + +func loadLocalRepo(config *PathConfig, path string) (*localRepo, error) { + currentRelativePath, _ := filepath.Rel(config.BasePath, path) + + l := &localRepo{ + BasePath: config.BasePath, + CurrentRelativePath: currentRelativePath, + } + l.StoredRelativePath() + l.RemoteId = l.config("remote-id") + return l, nil +} + +func LocalRepos(config *PathConfig) (repos []*localRepo, err error) { + err = filepath.Walk(config.BasePath, func(path string, info os.FileInfo, err error) error { + + if err != nil { + return err + } + if !info.IsDir() { + return nil + } + git, git_err := os.Lstat(filepath.Join(path, ".git")) + if git_err != nil { + return nil + } + if git.IsDir() { + path = filepath.Dir(path) + repo, _ := loadLocalRepo(config, path) + repos = append(repos, repo) + return filepath.SkipDir + } + return nil + }) + // TODO: move format up a level? + if err != nil { + return nil, fmt.Errorf("Failed finding local git repos for %s: %s", config.BasePath, err) + } + + return repos, nil +} + +/* +func RemoteRepos(config *PathConfig) []gitlabRepo { + +} +*/ diff --git a/service.go b/service.go new file mode 100644 index 0000000..41bb920 --- /dev/null +++ b/service.go @@ -0,0 +1,123 @@ +package gitlabsync + +import ( + "fmt" + gitlab_client "github.com/xanzy/go-gitlab" + "regexp" + "strings" + "sync" +) + +type Service interface { + RemoteRepos() ([]*remoteRepo, error) + CreateRepo(*localRepo) error + DeleteRepo(*remoteRepo) error + RenameRepo(*remoteRepo, string) error +} + +type Gitlab struct { + client *gitlab_client.Client + userConfig *UserConfig + pathConfig *PathConfig +} + +func NewGitlab(u *UserConfig, c *PathConfig) (*Gitlab, error) { + client := gitlab_client.NewClient(nil, c.AccessToken) + err := client.SetBaseURL(c.APIURL) + if err != nil { + return nil, err + } + return &Gitlab{client, u, c}, nil +} + +func (s *Gitlab) RemoteRepos() ([]*remoteRepo, error) { + WORKERS := 4 + pat := regexp.MustCompile(fmt.Sprintf("%s(?:/|$)", strings.Join(s.pathConfig.RemotePaths, "|"))) + totalPages := 1 + errors := make(chan error) + var allRepos []*remoteRepo + updateMutex := sync.Mutex{} + finished := make(chan *interface{}) + + for i := 1; i <= WORKERS; i++ { + go func(i int) { + defer func() { finished <- nil }() + tried := false + for page := i; !tried || i <= totalPages; i += WORKERS { + tried = true + perm := gitlab_client.GuestPermissions + opt := &gitlab_client.ListProjectsOptions{ + ListOptions: gitlab_client.ListOptions{ + PerPage: 100, + Page: page, + }, + MinAccessLevel: &perm, + } + ps, resp, err := s.client.Projects.ListProjects(opt) + if err != nil { + errors <- err + return + } + var pageRepos []*remoteRepo + for _, p := range ps { + if pat.MatchString(p.PathWithNamespace) { + pageRepos = append(pageRepos, &remoteRepo{ + Path: p.PathWithNamespace, + Id: fmt.Sprintf("%d", p.ID), + }) + } + } + updateMutex.Lock() + allRepos = append(allRepos, pageRepos...) + totalPages = resp.TotalPages + updateMutex.Unlock() + } + }(i) + } + running := WORKERS + for { + select { + case err := <-errors: + fmt.Print(err) + if err != nil { + return nil, err + } + case <-finished: + running -= 1 + if running == 0 { + return allRepos, nil + } + } + } +} + +func (s *Gitlab) CreateRepo(l *localRepo) error { + sep := strings.LastIndex(l.CurrentRelativePath, "/") + namespace, _, err := s.client.Namespaces.GetNamespace(l.CurrentRelativePath[:sep]) + if err == nil { + _, _, err = s.client.Projects.CreateProject(&gitlab_client.CreateProjectOptions{ + Name: gitlab_client.String(l.CurrentRelativePath[sep+1:]), + NamespaceID: gitlab_client.Int(namespace.ID), + }) + } + return err +} + +func (s *Gitlab) DeleteRepo(r *remoteRepo) error { + _, err := s.client.Projects.DeleteProject(r.Path) + return err +} + +func (s *Gitlab) RenameRepo(r *remoteRepo, path string) error { + sep := strings.LastIndex(path, "/") + namespace, _, err := s.client.Namespaces.GetNamespace(path[:sep]) + name := path[sep+1:] + if err == nil { + _, _, err = s.client.Projects.EditProject(r.Path, &gitlab_client.EditProjectOptions{ + Name: gitlab_client.String(name), + NamespaceID: gitlab_client.Int(namespace.ID), + }) + } + r.Path = path + return err +} diff --git a/service_test.go b/service_test.go new file mode 100644 index 0000000..c505440 --- /dev/null +++ b/service_test.go @@ -0,0 +1,10 @@ +package gitlabsync + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestImplementsService(t *testing.T) { + assert.Implements(t, (*Service)(nil), new(Gitlab)) +} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b200c23..0000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[aliases] -test=pytest - -[tool:pytest] diff --git a/setup.py b/setup.py deleted file mode 100644 index ca1a5f0..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -import codecs -import os - -from setuptools import setup - -here = os.path.abspath(os.path.dirname(__file__)) - -with codecs.open(os.path.join(here, "README.md"), encoding="utf-8") as f: - long_description = f.read() - -TEST_REQUIRES = ["pytest>=3.9", "pytest-docker", "grab", "requests"] -setup( - author="Oliver Bristow", - author_email="github+pypi@oliverbristow.co.uk", - name="gitlab-sync", - use_scm_version=True, - install_requires=["aiohttp", "click", "toml", "voluptuous"], - long_description=long_description, - long_description_content_type="text/markdown", - description="synchronise GitLab repositories", - setup_requires=["setuptools_scm", "wheel", "pytest-runner", "attrs"], - tests_require=TEST_REQUIRES, - extras_require={"test": TEST_REQUIRES}, - entry_points={"console_scripts": ["gitlab-sync = gitlab_sync.cli:main"]}, - packages=["gitlab_sync"], - python_requires=">=3.6", -) diff --git a/strategy.go b/strategy.go new file mode 100644 index 0000000..915263d --- /dev/null +++ b/strategy.go @@ -0,0 +1,124 @@ +package gitlabsync + +import ( + "fmt" +) + +type SyncRun struct { + id int + userConfig *UserConfig + pathConfig *PathConfig +} + +type repoPair struct { + local *localRepo + remote *remoteRepo +} + +// really a sync/strat thing, and put it in there rather than on localRepo, really inside a loop once +func branchSyncInfo(l *localRepo, branch string) (ahead uint, behind uint, err error) { + // could have error like non-common history + // git rev-list --left-right --count {local}..{remote} + return +} + +// plan for syncing repo metadata + default branch +type metaSyncPlan struct { + // need pair + DeleteResolve []repoPair // can't happen as remote deltes not detectable + DeleteLocal []*localRepo + DeleteRemote []*remoteRepo // Not currently detectable + // need pair + RenameResolve []repoPair + RenameLocal []repoPair + RenameRemote []repoPair + // need pair + CreateResolve []repoPair + CreateRemote []*localRepo + CreateLocal []*remoteRepo +} + +// separated out for ease of testing +func newMetaSyncPlan(locals []localRepo, remotes []remoteRepo) (plan metaSyncPlan) { + pairs := make(map[string]repoPair) + // work out what has to be created+deleted, and find pairs + { + localMap := make(map[string]*localRepo) + createRemote := make(map[string]*localRepo) + for _, local := range locals { + if len(local.RemoteId) != 0 { + localMap[local.RemoteId] = &local + } else { + createRemote[local.CurrentRelativePath] = &local + } + } + for _, remote := range remotes { + local, exists := localMap[remote.Id] + if !exists { + if local, exists = createRemote[remote.Path]; exists { + plan.CreateResolve = append(plan.CreateResolve, repoPair{local, &remote}) + delete(createRemote, remote.Path) + } else { + plan.CreateLocal = append(plan.CreateLocal, &remote) + } + continue + } + pairs[remote.Id] = repoPair{local, &remote} + // TODO: pop from local map, then remaining ones are for DeleteLocal + delete(localMap, remote.Id) + } + // case in these next two loops which should produce CreateResolve? humm, could also be "paired DeleteLocal/CreateRemote" + for _, local := range localMap { + plan.DeleteLocal = append(plan.DeleteLocal, local) + } + for _, remote := range createRemote { + plan.CreateRemote = append(plan.CreateRemote, remote) + } + } + // work out renames + for _, pair := range pairs { + if pair.local.CurrentRelativePath != pair.remote.Path { + if pair.local.storedRelativePath == pair.remote.Path { + plan.RenameRemote = append(plan.RenameRemote, pair) + } else if pair.local.storedRelativePath == pair.local.CurrentRelativePath { + plan.RenameLocal = append(plan.RenameLocal, pair) + } else { + plan.RenameResolve = append(plan.RenameResolve, pair) + } + } + } + + return +} + +// probably want to roll use of SyncRun into plan, e.g. make plan return a struct +// that a strategy can use +func newMetaSyncPlanz(s *SyncRun) { + service, err := NewGitlab(s.userConfig, s.pathConfig) + if err != nil { + panic(err) + } + fmt.Print("About to get repos\n") + remotes, err := service.RemoteRepos() + if err != nil { + panic(err) + } + for _, remote := range remotes { + fmt.Printf("%#v\n", *remote) + } + // is there a way to logically enumerate the scenarios and classify them? + // itertools.product(local_states, remote_states) + // if (local.stored_path == ...): assert plan.thing + // DeleteLocal if id in local but not remote + // DeleteRemote if ... (no way to know without state between runs) + // DeleteResolve if ... removed locally updated remotely + + // RenameLocal if stored path is same as current, but path in remote is new + // RenameRemote if stored path is different, but path in remote is stored + // RenameResolve if stored path is different, and path in remote matches neither + // what about when both places move? need to update stored path + + // CreateRemote if local repo with no stored remote id + // CreateLocal if no matching repo id in locals + // CreateResolve is ones that would have overlapping path in CreateLocal & CreateRemote +} diff --git a/strategy_test.go b/strategy_test.go new file mode 100644 index 0000000..e98c88a --- /dev/null +++ b/strategy_test.go @@ -0,0 +1,83 @@ +package gitlabsync + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewMetaSyncPlan(t *testing.T) { + // could be a whole log of for loops that permutate a field + paths := []string{"", "a", "b"} + remoteIds := []string{"", "1", "2"} + ids := []string{"1", "2"} + + // FIXME: path can legitimately be "", should have a nillable one? + + // enumerate permutations of local repos + local := localRepo{BasePath: "test-base"} + for _, currentRelativePath := range paths { + local.CurrentRelativePath = currentRelativePath + for _, storedRelativePath := range paths { + local.storedRelativePath = storedRelativePath + for _, remoteId := range remoteIds { + local.RemoteId = remoteId + + // and enumerate permutations of remote repos + remote := remoteRepo{} + for _, id := range ids { + remote.Id = id + for _, remotePath := range paths { + remote.Path = remotePath + plan := newMetaSyncPlan([]localRepo{local}, []remoteRepo{remote}) + + if remote.Id == local.RemoteId { + // we have a matched (local, remote) pair, check for moves + if remote.Path == local.CurrentRelativePath { + // no change + assert.Equal(t, plan, metaSyncPlan{}) + } else if local.storedRelativePath == remote.Path { + // move remote + assert.Equal(t, plan, metaSyncPlan{ + RenameRemote: []repoPair{repoPair{&local, &remote}}, + }) + } else if local.CurrentRelativePath == local.storedRelativePath { + // remote path changed + assert.Equal(t, plan, metaSyncPlan{ + RenameLocal: []repoPair{repoPair{&local, &remote}}, + }) + } else { + // rename resolve if CurrentRelativePath != storedRelativePath + + // remote.Path != CurrentRelativePath + assert.Equal(t, plan, metaSyncPlan{ + RenameResolve: []repoPair{repoPair{&local, &remote}}, + }) + } + } else { + // creates or deletes are required + if local.CurrentRelativePath == remote.Path { // storedRelativePath won't exist here + // conflict + assert.Equal(t, plan, metaSyncPlan{ + CreateResolve: []repoPair{repoPair{&local, &remote}}, + }) + } else if local.RemoteId == "" { + // remote needs to be created from local + assert.Equal(t, plan, metaSyncPlan{ + CreateRemote: []*localRepo{&local}, + CreateLocal: []*remoteRepo{&remote}, + }) + } else { + // there was a remote, but not now, so delete local + assert.Equal(t, plan, metaSyncPlan{ + DeleteLocal: []*localRepo{&local}, + CreateLocal: []*remoteRepo{&remote}, + }) + } + // there is no way to detect the need for a remote delete due to state being in the deleted + // repo. It seems reasonable to avoid automatic remote deletes anyway. + } + } + } + } + } + } +} From 7cb73a44490b7c3ec478fde4f5987a6fee047677 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 17 Nov 2019 00:10:46 +0000 Subject: [PATCH 2/2] WIP: golang refactor --- README.md | 204 +++++++++++++++++++++++++++++++++++++++- cmd/gitlab-sync/main.go | 136 +++++++++++++++++++++++---- config.go | 1 + go.mod | 2 + repository.go | 136 ++++++++++++++++----------- service.go | 45 +++++---- strategy.go | 44 ++++----- 7 files changed, 449 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index 00714dc..9de4c65 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ # gitlab-sync This provides the gitlab-sync tool which clones GitLab and updates repositories. +## Ideas + +~~Providers are to their collections, as a~~ collections are to their repositories, as a repositories are to their branches. + +Do not allow multiple ways to do the same thing. + + + ## Config You will need to have [SSH access configured for GitLab](https://docs.gitlab.com/ee/ssh/), and @@ -34,10 +42,192 @@ strategy = "mirror" ``` +```yaml +providers: + gitlab-tracr: + service: gitlab + api-token: "iujasduniuuhu3489djf" + dummy: + # this does not create/delete/move repos from anywhere + # FIXME: this is really just the null provider + collection config + service: dummy + +collections: + - provider: gitlab-tracr + path: tracr/pcs/ # path always stripped + directory: "~/src/gitlab.com/tracr/pcs" + repositories: + local: none/pull/sync - (), (copy), (create+delete+move) + remote: none/pull/sync - (), (create), (create+delete+move) + branches: + some-name-*: + local: null/pull/sync + remote: null/pull/sync + commits: + local: null/fast-forward/merge/force/rebase + remote: null/fast-forward/merge/force/rebase + tags: # probably leave undefined in first implementation + local: null/pull/sync + remote: null/pull/sync +# branch-reconciliation: +# resolution policy can be per run, e.g. revert, ask, manual, force +# working-tree policy: +# cleaning policy can be per run, e.g. abort, ask, manual, purge, stash +# # branch level plugin +# plugins: +# run-on-commit: false +# # collection level plugin +# plugins: +# run-on-commit: true # install post-commit hook - when to install? +# poll-interval: "* * * * *" # use https://github.com/robfig/cron +# could have config to apply locally per-repo (e.g. merge/diff strats) + +options: + log: + file: "/var/log/oof" + size-limit: 1M + count-limit: 5 +``` + +### Repository and branch policies + +These policies are used to determine what to do when changes are detected to the names/existence in the opposite repositories or branches. + +A repository is dirty if it needs action, or it's branches are dirty. + +A branch is dirty if it needs action, or it's commits are dirty? A branch is commits... + +#### null + +This ignores the changes. + +#### pull + +New repositories or branches will be created, but will fail if they already exist. + +#### sync + +A branch is considered deleted if one with the same name does not exist. How is a branch considered created? How does this fit with `git fetch --prune`? Sync policy is really to stop a mess of missing branches, but could this be done with some other policy combination with clean policy application on branches? + +### Commit policies + +The branch policy is defined on a collection, and tells git-sync what to do to branches during a run. + +The patterns match the local branch name? What about the remote name? These can differ from the upstream... upstream is used in operations so should be the priority, also lets users change their upstreams for the behaviour. Just use the commit policies to set up local copies? Could have branch "functions"/plugins for complex behaviour. + +The commit strategy is a branch reconciliation strategy when both exist. This strategy defines: + +- equality (i.e. no work needed) +- reconciliation (what to try doing when work needed + +As this has two sides, the operations must be compatible, e.g. dual merge doesn't work as never able to reach equality after diverging. Given two commits, a plan is made by looking at the operations. + +Compatability (local, remote): + +* none - none -> [] + * none - fast-forward -> [--ff-only] +* fast-toward - fast-forward (any goes first? stop when one succeeds) -> [] +* merge - fast-forward -> [FFAFAP(local, remote)] +* merge - merge -> which merge goes first? + +#### none + +#### fast-foward + +#### merge + +#### force + +#### rebase + +| Policy | Description | Deletion | Notes | +| ----------- | ------------------------------------------------------------ | ------------------------------------------------------------ | ----- | +| none | do not do anything when there are changes on the corresponding local/remote branch | no branches will be created or deleted | | +| fast-foward | fetch changes and attempt to fast-forward them in | branches will be deleted if they are not present remotely but exist in some branch | | +| merge | fetch changes and attempt to merge them in | | | +| rebase | fetch changes and rebase onto them | | | +| force | fetch changes and use them | | | + +### Patterns + +You can use hidden YAML nodes, anchors, and [merge keys](https://yaml.org/type/merge.html) to reduce repition in your configuration. + +```yaml +.my-pattern: &my-pattern + repositories: + local: none +collections: + - provider: + path: + directory: + <<: *my-pattern +``` + +#### mirroring + +If you set the local policy for repositories and + +```yaml +.mirror: &mirror + repositories: + local: pull + branches: + *: + local: pull + commits: + local: force +``` + +#### shared repositories + +```yaml +.shared: &shared + repositories: + local: sync + branches: + # the local master follows the remote + master: + local: pull + commits: + local: fast-forward + # branches following the _username-*_ pattern will be forced to match the local copy + username-wip-*: + remote: sync + commits: + remote: force + plugin: + # automatically create/update a MR the subect + body set by head commit + gitlab-mr: + target: master +``` + +#### local fork + +You can change the upstream of a branch? how does that play into the cleaning up? + +```yaml +.local-fork: &local-fork + repositories: + local: pull + branches: + forked-*: + commits: + # gets and rebase onto the tracked upstream + local: rebase +``` + + ## Usage + ``` -$ gitlab-sync local-update +=all|tree|subtree|current +$ git-sync [all|tree|current] ... +$ git-sync pull [--all|--head|--branch=] +$ git-sync checkout [--changes=drop|warn|] +$ git-sync policy --set=mirror +$ # git-sync log --since= --until= --type= +$ git-sync directories --null ``` ### Strategies @@ -56,6 +246,18 @@ in using something like [`ag`](https://github.com/ggreer/the_silver_searcher). The local copies should not be modified by users. +### sync + +Options: + +* Fast-forward-only +* Rebase-only + +## Extra + +* nested strategies, e.g. mirror org and sync subgroup(s) - how are subgroup moves tracked? dangerous to squash differences if a branch moves, e.g. from sync (with work on) to a mirror that forces changes +* list git directories + ## To do * separate out packages so it's clear where things are coming from diff --git a/cmd/gitlab-sync/main.go b/cmd/gitlab-sync/main.go index 2157143..b217ffc 100644 --- a/cmd/gitlab-sync/main.go +++ b/cmd/gitlab-sync/main.go @@ -1,14 +1,112 @@ package main +/* +# use YAML instead of TOML to allow easy provider info reuse +.GitLab: &GitLab + provider: + - name: gitlab + - parameters: + access-token: heloo +UserConfig: + PathConfig: + - base-path: tracr/ + local-prefix: /home/obristow/code/ + strategy: pull + <<: *.GitLab + provider: + - name: gitlab + parameters: + - access-token: heloo + branches: + master: + strategy: mirror + +Each repository found from the sources (including the local provider which uses the local-prefix) is bundled together to config-wide reconciliation: +struct Info { + Provider *RepoService, // object implementing RepoService interface + Id str, // id of the repo for the given provider + LocalBasePath str, // base of repo locally according to config + LocalRelativePath str, // platform/event-service + RemotePath str, // tracr/pcs/plafrom/event-service + RemoteUrl str, // git@github.com:/some/path + Strategy str, // "pull" (local) + "push" (remote), "reset" (local) + "push" (remote) +} + +// Read is implicit for all? +Service.Readable("branch-1") -> True +Service.Writable("master") -> False +join repos on (LocalBasePath, id) - will need to check config to avoid duplicate paths; what about nested paths e.g. groups? - just have to error nicely +anything with no existing local repo gets created? + +type RefPolicy int + +const ( + RefNull = RefPolicy iota + RefPull + RefSync +) + +struct RefPolicyInfo { + Local RefPolicy + Remote RefPolicy +} + +type CommitPolicy int + +const ( + CommitNull CommitPolicy = iota + CommitFastForward + CommitMerge + CommitForce + CommitRebase +) + +struct CommitPolicyInfo { + Local CommitPolicy + Remote CommitPolicy +} + +struct Collection { + Provider str + Path str + Directory str + Tags RefPolicyInfo + Branches map[str]RefPolicyInfo + Commits CommitPolicyInfo +// Plugins map[str]struct{} +} + +struct Config { + Collections []Collection + Options [str]struct{} +} + +*/ import ( "fmt" - // "github.com/spf13/cobra" - "github.com/Code0x58/gitlab-sync" - "sync" + + gitlabsync "github.com/Code0x58/gitlab-sync" ) func main() { - var c, err = gitlabsync.LoadConfig() + c, err := gitlabsync.LoadConfig() + if err != nil { + panic(err) + } + fmt.Println(c) + + locals, _ := gitlabsync.LocalRepos(&c.PathConfigs[0]) + fmt.Println(locals) + gl, err := gitlabsync.NewGitlab(c, &c.PathConfigs[0]) + remotes, err := gl.RemoteRepos() + if err != nil { + panic(err) + } + for _, r := range remotes { + fmt.Println(*r) + } + + /** if err != nil { panic(fmt.Errorf("Fatal error in config: %s\n", err)) } @@ -21,24 +119,21 @@ func main() { go func() { defer wg.Done() sync := <-syncs - fmt.Printf("%#v\n", sync.pathConfig) - fmt.Printf("access-token: %s\n", sync.pathConfig.AccessToken) + fmt.Printf("%#v\n", sync.PathConfig) + fmt.Printf("access-token: %s\n", sync.PathConfig.AccessToken) // gitlabsync.LocalRepos(&pathConfig) - var remoteRepos []*remoteRepo - { - service, err := NewGitlab(s.userConfig, s.pathConfig) - if err != nil { - panic(err) - } - fmt.Print("About to get repos\n") - remoteRepos, err = service.RemoteRepos() - if err != nil { - panic(err) - } - for _, remote := range remotes { - fmt.Printf("%#v\n", *remote) - } + service, err := gitlabsync.NewGitlab(sync.UserConfig, sync.PathConfig) + if err != nil { + panic(err) + } + fmt.Print("About to get repos\n") + remotes, err := service.RemoteRepos() + if err != nil { + panic(err) + } + for _, remote := range remotes { + fmt.Printf("%#v\n", *remote) } gitlabsync.newMetaSyncPlan(locals, remotes) }() @@ -50,4 +145,5 @@ func main() { // starve and wait for the pool to finish close(syncs) wg.Wait() + **/ } diff --git a/config.go b/config.go index f9900d8..84dac58 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ import ( type UserConfig struct { // naming of the last two should be improvable + // probably want a semaphore here for work limits (like hitting GitLab) PathConfigs []PathConfig `toml:"path-config"` MaxParallelSyncs uint `toml:"max-parallel-syncs"` MaxParallelSyncOperations uint `toml:"max-parallel-sync-operationns"` diff --git a/go.mod b/go.mod index aaf5150..1e76c13 100644 --- a/go.mod +++ b/go.mod @@ -11,3 +11,5 @@ require ( golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect google.golang.org/appengine v1.4.0 // indirect ) + +go 1.13 diff --git a/repository.go b/repository.go index 12bb565..790da1c 100644 --- a/repository.go +++ b/repository.go @@ -4,89 +4,117 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" ) -type localRepo struct { - BasePath string // base path on local system - CurrentRelativePath string // current relative path on local system - storedRelativePath string // previous relative path on local system - RemoteId string // id of project in the remote service +// TODO: make an interface for +// could use this with an interface like Get and Set +// then again, if this + remote are an interface, then localRepoInfo could be used in the generic methods like "Service.Create(local)" or CreateLocal(remote, config) +type localRepoInfo struct { + // could be BasePath, e.g. /usr/obristow/code/, or git://git.github.com/ + localPath string // path in filesystem - don't need more assuming users never move local directories relative to their base? + Id string `json:"id"` // project Id + AbsolutePath string `json:"absolute-path"` // e.g. org/group/project + RelativePath string `json:"relative-path"` // e.g. group/project + // strategy could be inferred from the status of each repo, like "canonical" or "source", "downstream" + Strategy string `json:"strategy"` // this would imply both can record/report a stragegy; providers chould set the strat, e.g. from local system config } -type remoteRepo struct { - Path string // path of the repo in service - Id string // id of project in service +func LoadLocalRepo(path string) (l *localRepoInfo, err error) { + // load a configured repo from the local filesystem + l.localPath = path + // TODO: get exceptions + l.Id = l.config("id") + l.AbsolutePath = l.config("absolute-path") + l.RelativePath = l.config("relative-path") + l.Strategy = l.config("strategy") + return } // Run a git command on the local repository -func (l *localRepo) git(args ...string) *exec.Cmd { - path := filepath.Join(l.BasePath, l.CurrentRelativePath) +func (l *localRepoInfo) git(args ...string) *exec.Cmd { all_args := make([]string, len(args)+3) all_args[0] = "git" all_args[1] = "-C" - all_args[2] = path + all_args[2] = l.localPath copy(all_args[3:], args) return exec.Command("git", all_args...) } // Get a config item from the local repository, or an empty string -func (l *localRepo) config(name string) string { - item := fmt.Sprintf("gitlab-sync.%s", name) +func (l *localRepoInfo) config(name string) string { + item := fmt.Sprintf("git-sync.%s", name) out, _ := l.git("config", "--local", item).Output() return string(out) } -func (l *localRepo) setConfig(name string, value string) error { - item := fmt.Sprintf("gitlab-sync.%s", name) - _, err := l.git("config", "--local", item, value).Output() +func (l *localRepoInfo) setConfig(name string, value string) (err error) { + item := fmt.Sprintf("git-sync.%s", name) + _, err = l.git("config", "--local", item, value).Output() // TODO(obristow): think about a panic in this situation - return err + return } -// Return the relative path that was last seen -func (l *localRepo) StoredRelativePath() string { - if len(l.storedRelativePath) == 0 { - l.storedRelativePath = l.config("relative-path") - } - return l.storedRelativePath +func (l *localRepoInfo) save() { + // TODO: return error or panic on failure + l.setConfig("id", l.Id) + l.setConfig("absolute-path", l.AbsolutePath) // can infer from base+prefix rules + path + l.setConfig("relative-path", l.RelativePath) // path - base + l.setConfig("strategy", l.Strategy) } -// create the local repository -func (l *localRepo) Create() error { - _, err := l.git("init", ".").Output() - if err != nil { - return err - } - l.setConfig("relative-path", l.CurrentRelativePath) - l.setConfig("remote-id", l.RemoteId) - return nil +// TODO: think about converting this to an interface so different providers can do it in their own package? +// probably not much point as have to compile into code +type remoteRepoInfo struct { + Id string `json:"id"` + AbsolutePath string `json:"absolute-path"` } -// rename the local repository -func (l *localRepo) Rename(relativePath string) error { - // TODO(obristow): move to the given relative path - /* - out, err := l.git("init", ".").Output() - l.CurrentRelativePath = relativePath - l.setConfig("relative-path", relativePath) - */ - return nil +func relativePath(path, prefix string) (p string, err error) { + // return the path relative to the given prefix, or an error if it does not start with the prefix + failure := fmt.Errorf("No common prefix!") + for i := range prefix { + if path[i] != prefix[i] { + err = failure + } + } + if len(prefix) == len(path) { + p = "" + } else if path[len(prefix)] == '/' { + p = path[len(prefix)+1:] + } else { + err = failure + } + return } -func loadLocalRepo(config *PathConfig, path string) (*localRepo, error) { - currentRelativePath, _ := filepath.Rel(config.BasePath, path) +func (r *remoteRepoInfo) RelativePath(prefix string) (path string, err error) { + // return the path relative to the given prefix, or an error if it does not start with the prefix + return relativePath(r.AbsolutePath, prefix) +} - l := &localRepo{ - BasePath: config.BasePath, - CurrentRelativePath: currentRelativePath, +func NewLocalRepo(base string, r *remoteRepoInfo, strip string) (l *localRepoInfo, err error) { + // return a local repo + localRelative, err := relativePath(r.AbsolutePath, strip) + if err != nil { + return + } + l.localPath = path.Join(base, localRelative) + l.Id = r.Id + l.AbsolutePath = r.AbsolutePath + l.RelativePath = localRelative + l.Strategy = "mirror" + err = l.git("init", ".").Run() + if err != nil { + err = fmt.Errorf("Unable to create new repository at %s for %s: %s", l.localPath, l.AbsolutePath, err) + return nil, err } - l.StoredRelativePath() - l.RemoteId = l.config("remote-id") - return l, nil + l.save() + return } -func LocalRepos(config *PathConfig) (repos []*localRepo, err error) { +func LocalRepos(config *PathConfig) (repos []*localRepoInfo, err error) { err = filepath.Walk(config.BasePath, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -101,7 +129,7 @@ func LocalRepos(config *PathConfig) (repos []*localRepo, err error) { } if git.IsDir() { path = filepath.Dir(path) - repo, _ := loadLocalRepo(config, path) + repo, _ := LoadLocalRepo(path) repos = append(repos, repo) return filepath.SkipDir } @@ -114,9 +142,3 @@ func LocalRepos(config *PathConfig) (repos []*localRepo, err error) { return repos, nil } - -/* -func RemoteRepos(config *PathConfig) []gitlabRepo { - -} -*/ diff --git a/service.go b/service.go index 41bb920..f476cc5 100644 --- a/service.go +++ b/service.go @@ -2,17 +2,18 @@ package gitlabsync import ( "fmt" - gitlab_client "github.com/xanzy/go-gitlab" "regexp" "strings" "sync" + + gitlab_client "github.com/xanzy/go-gitlab" ) type Service interface { - RemoteRepos() ([]*remoteRepo, error) - CreateRepo(*localRepo) error - DeleteRepo(*remoteRepo) error - RenameRepo(*remoteRepo, string) error + RemoteRepos() ([]*remoteRepoInfo, error) + CreateRepo(*remoteRepoInfo) error + DeleteRepo(*remoteRepoInfo) error + RenameRepo(*remoteRepoInfo, string) error } type Gitlab struct { @@ -21,6 +22,7 @@ type Gitlab struct { pathConfig *PathConfig } +// TODO(obristow): review. if only have RemoteRepos method, then this can avoid the struct func NewGitlab(u *UserConfig, c *PathConfig) (*Gitlab, error) { client := gitlab_client.NewClient(nil, c.AccessToken) err := client.SetBaseURL(c.APIURL) @@ -30,12 +32,12 @@ func NewGitlab(u *UserConfig, c *PathConfig) (*Gitlab, error) { return &Gitlab{client, u, c}, nil } -func (s *Gitlab) RemoteRepos() ([]*remoteRepo, error) { +func (s *Gitlab) RemoteRepos() ([]*remoteRepoInfo, error) { WORKERS := 4 pat := regexp.MustCompile(fmt.Sprintf("%s(?:/|$)", strings.Join(s.pathConfig.RemotePaths, "|"))) totalPages := 1 errors := make(chan error) - var allRepos []*remoteRepo + var allRepos []*remoteRepoInfo updateMutex := sync.Mutex{} finished := make(chan *interface{}) @@ -58,12 +60,12 @@ func (s *Gitlab) RemoteRepos() ([]*remoteRepo, error) { errors <- err return } - var pageRepos []*remoteRepo + var pageRepos []*remoteRepoInfo for _, p := range ps { if pat.MatchString(p.PathWithNamespace) { - pageRepos = append(pageRepos, &remoteRepo{ - Path: p.PathWithNamespace, - Id: fmt.Sprintf("%d", p.ID), + pageRepos = append(pageRepos, &remoteRepoInfo{ + AbsolutePath: p.PathWithNamespace, + Id: fmt.Sprintf("%d", p.ID), }) } } @@ -91,33 +93,36 @@ func (s *Gitlab) RemoteRepos() ([]*remoteRepo, error) { } } -func (s *Gitlab) CreateRepo(l *localRepo) error { - sep := strings.LastIndex(l.CurrentRelativePath, "/") - namespace, _, err := s.client.Namespaces.GetNamespace(l.CurrentRelativePath[:sep]) +// TODO(obristow): review. delete (leaving remote as source of repos) +func (s *Gitlab) CreateRepo(l *remoteRepoInfo) error { + sep := strings.LastIndex(l.AbsolutePath, "/") + namespace, _, err := s.client.Namespaces.GetNamespace(l.AbsolutePath[:sep]) if err == nil { _, _, err = s.client.Projects.CreateProject(&gitlab_client.CreateProjectOptions{ - Name: gitlab_client.String(l.CurrentRelativePath[sep+1:]), + Name: gitlab_client.String(l.AbsolutePath[sep+1:]), NamespaceID: gitlab_client.Int(namespace.ID), }) } return err } -func (s *Gitlab) DeleteRepo(r *remoteRepo) error { - _, err := s.client.Projects.DeleteProject(r.Path) +// TODO(obristow): review. delete (leaving remote as source of repos) +func (s *Gitlab) DeleteRepo(r *remoteRepoInfo) error { + _, err := s.client.Projects.DeleteProject(r.AbsolutePath) return err } -func (s *Gitlab) RenameRepo(r *remoteRepo, path string) error { +// TODO(obristow): review. delete (leaving remote as source of repos) +func (s *Gitlab) RenameRepo(r *remoteRepoInfo, path string) error { sep := strings.LastIndex(path, "/") namespace, _, err := s.client.Namespaces.GetNamespace(path[:sep]) name := path[sep+1:] if err == nil { - _, _, err = s.client.Projects.EditProject(r.Path, &gitlab_client.EditProjectOptions{ + _, _, err = s.client.Projects.EditProject(r.AbsolutePath, &gitlab_client.EditProjectOptions{ Name: gitlab_client.String(name), NamespaceID: gitlab_client.Int(namespace.ID), }) } - r.Path = path + r.AbsolutePath = path return err } diff --git a/strategy.go b/strategy.go index 915263d..49e3091 100644 --- a/strategy.go +++ b/strategy.go @@ -6,17 +6,17 @@ import ( type SyncRun struct { id int - userConfig *UserConfig - pathConfig *PathConfig + UserConfig *UserConfig + PathConfig *PathConfig } type repoPair struct { - local *localRepo - remote *remoteRepo + local *localRepoInfo + remote *remoteRepoInfo } // really a sync/strat thing, and put it in there rather than on localRepo, really inside a loop once -func branchSyncInfo(l *localRepo, branch string) (ahead uint, behind uint, err error) { +func branchSyncInfo(l *localRepoInfo, branch string) (ahead uint, behind uint, err error) { // could have error like non-common history // git rev-list --left-right --count {local}..{remote} return @@ -26,38 +26,39 @@ func branchSyncInfo(l *localRepo, branch string) (ahead uint, behind uint, err e type metaSyncPlan struct { // need pair DeleteResolve []repoPair // can't happen as remote deltes not detectable - DeleteLocal []*localRepo - DeleteRemote []*remoteRepo // Not currently detectable + DeleteLocal []*localRepoInfo + DeleteRemote []*remoteRepoInfo // Not currently detectable // need pair RenameResolve []repoPair RenameLocal []repoPair RenameRemote []repoPair // need pair CreateResolve []repoPair - CreateRemote []*localRepo - CreateLocal []*remoteRepo + CreateRemote []*localRepoInfo + CreateLocal []*remoteRepoInfo } // separated out for ease of testing -func newMetaSyncPlan(locals []localRepo, remotes []remoteRepo) (plan metaSyncPlan) { +func newMetaSyncPlan(locals []localRepoInfo, remotes []remoteRepoInfo) (plan metaSyncPlan) { pairs := make(map[string]repoPair) // work out what has to be created+deleted, and find pairs { - localMap := make(map[string]*localRepo) - createRemote := make(map[string]*localRepo) + localMap := make(map[string]*localRepoInfo) + createRemote := make(map[string]*localRepoInfo) for _, local := range locals { - if len(local.RemoteId) != 0 { - localMap[local.RemoteId] = &local + if len(local.Id) != 0 { + localMap[local.Id] = &local } else { - createRemote[local.CurrentRelativePath] = &local + // FIXME: this is miles off because ... + createRemote[local.RelativePath] = &local } } for _, remote := range remotes { local, exists := localMap[remote.Id] if !exists { - if local, exists = createRemote[remote.Path]; exists { + if local, exists = createRemote[remote.AbsolutePath]; exists { plan.CreateResolve = append(plan.CreateResolve, repoPair{local, &remote}) - delete(createRemote, remote.Path) + delete(createRemote, remote.AbsolutePath) } else { plan.CreateLocal = append(plan.CreateLocal, &remote) } @@ -77,10 +78,11 @@ func newMetaSyncPlan(locals []localRepo, remotes []remoteRepo) (plan metaSyncPla } // work out renames for _, pair := range pairs { - if pair.local.CurrentRelativePath != pair.remote.Path { - if pair.local.storedRelativePath == pair.remote.Path { + // FIXME(obristow): slammed the shit out of these branches just to compile + if pair.local.AbsolutePath != pair.remote.AbsolutePath { + if pair.local.AbsolutePath == pair.remote.AbsolutePath { plan.RenameRemote = append(plan.RenameRemote, pair) - } else if pair.local.storedRelativePath == pair.local.CurrentRelativePath { + } else if pair.local.AbsolutePath == pair.local.AbsolutePath { plan.RenameLocal = append(plan.RenameLocal, pair) } else { plan.RenameResolve = append(plan.RenameResolve, pair) @@ -94,7 +96,7 @@ func newMetaSyncPlan(locals []localRepo, remotes []remoteRepo) (plan metaSyncPla // probably want to roll use of SyncRun into plan, e.g. make plan return a struct // that a strategy can use func newMetaSyncPlanz(s *SyncRun) { - service, err := NewGitlab(s.userConfig, s.pathConfig) + service, err := NewGitlab(s.UserConfig, s.PathConfig) if err != nil { panic(err) }