From d58dbebce24641247e8e181223e079425b282f93 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 23 Nov 2025 15:57:41 +0100 Subject: [PATCH] Add repo dir sync [skip ci] --- repo_dir_sync.py | 363 +++++++++++++++++++++++++++++++++++++++++++++++ sync.py | 6 + 2 files changed, 369 insertions(+) create mode 100644 repo_dir_sync.py create mode 100644 sync.py diff --git a/repo_dir_sync.py b/repo_dir_sync.py new file mode 100644 index 00000000..874a362f --- /dev/null +++ b/repo_dir_sync.py @@ -0,0 +1,363 @@ +# -*- coding: utf-8 -*- +import glob +import os +import shutil +from subprocess import Popen, PIPE +import re +import sys +from typing import List, Optional, Sequence +import platform + + +def popen(cmd): + shell = platform.system() != "Windows" + p = Popen(cmd, shell=shell, stdin=PIPE, stdout=PIPE) + return p + + +def call(cmd): + p = popen(cmd) + return p.stdout.read().decode("utf-8") + + +def execute(cmd, exceptionOnError=True): + """ + :param cmd: the command to execute + :param exceptionOnError: if True, raise on exception on error (return code not 0); if False return + whether the call was successful + :return: True if the call was successful, False otherwise (if exceptionOnError==False) + """ + p = popen(cmd) + p.wait() + success = p.returncode == 0 + if exceptionOnError: + if not success: + raise Exception("Command failed: %s" % cmd) + else: + return success + + +def gitLog(path, arg): + oldPath = os.getcwd() + os.chdir(path) + lg = call("git log --no-merges " + arg) + os.chdir(oldPath) + return lg + + +def gitCommit(msg): + with open(COMMIT_MSG_FILENAME, "wb") as f: + f.write(msg.encode("utf-8")) + gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME) + + +def gitCommitWithMessageFromFile(commitMsgFilename): + if not os.path.exists(commitMsgFilename): + raise FileNotFoundError(f"{commitMsgFilename} not found in {os.path.abspath(os.getcwd())}") + os.system(f"git commit --file={commitMsgFilename}") + os.unlink(commitMsgFilename) + + +COMMIT_MSG_FILENAME = "commitmsg.txt" + + +class OtherRepo: + SYNC_COMMIT_ID_FILE_LIB_REPO = ".syncCommitId.remote" + SYNC_COMMIT_ID_FILE_THIS_REPO = ".syncCommitId.this" + SYNC_COMMIT_MESSAGE = f"Updated %s sync commit identifiers" + SYNC_BACKUP_DIR = ".syncBackup" + + def __init__(self, name, branch, pathToLib): + self.pathToLibInThisRepo = os.path.abspath(pathToLib) + if not os.path.exists(self.pathToLibInThisRepo): + raise ValueError(f"Repository directory '{self.pathToLibInThisRepo}' does not exist") + self.name = name + self.branch = branch + self.libRepo: Optional[LibRepo] = None + + def isSyncEstablished(self): + return os.path.exists(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO)) + + def lastSyncIdThisRepo(self): + with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_THIS_REPO), "r") as f: + commitId = f.read().strip() + return commitId + + def lastSyncIdLibRepo(self): + with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO), "r") as f: + commitId = f.read().strip() + return commitId + + def gitLogThisRepoSinceLastSync(self): + lg = gitLog(self.pathToLibInThisRepo, '--name-only HEAD "^%s" .' % self.lastSyncIdThisRepo()) + lg = re.sub(r'commit [0-9a-z]{8,40}\n.*\n.*\n\s*\n.*\n\s*(\n.*\.syncCommitId\.(this|remote))+', r"", lg, flags=re.MULTILINE) # remove commits with sync commit id update + indent = " " + lg = indent + lg.replace("\n", "\n" + indent) + return lg + + def gitLogLibRepoSinceLastSync(self, libRepo: "LibRepo"): + syncIdFile = os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO) + if not os.path.exists(syncIdFile): + return "" + with open(syncIdFile, "r") as f: + syncId = f.read().strip() + lg = gitLog(libRepo.libPath, '--name-only HEAD "^%s" .' % syncId) + lg = re.sub(r"Sync (\w+)\n\s*\n", r"Sync\n\n", lg, flags=re.MULTILINE) + indent = " " + lg = indent + lg.replace("\n", "\n" + indent) + return "\n\n" + lg + + def _userInputYesNo(self, question) -> bool: + result = None + while result not in ("y", "n"): + result = input(question + " [y|n]: ").strip() + return result == "y" + + def pull(self, libRepo: "LibRepo"): + """ + Pulls in changes from this repository into the lib repo + """ + # switch to branch in lib repo + os.chdir(libRepo.rootPath) + execute("git checkout %s" % self.branch) + + # check if the branch contains the commit that is referenced as the remote commit + remoteCommitId = self.lastSyncIdLibRepo() + remoteCommitExists = execute("git rev-list HEAD..%s" % remoteCommitId, exceptionOnError=False) + if not remoteCommitExists: + if not self._userInputYesNo(f"\nWARNING: The referenced remote commit {remoteCommitId} does not exist " + f"in your {self.libRepo.name} branch '{self.branch}'!\nSomeone else may have " + f"pulled/pushed in the meantime.\nIt is recommended that you do not continue. " + f"Continue?"): + return + + # check if this branch is clean + lgLib = self.gitLogLibRepoSinceLastSync(libRepo).strip() + if lgLib != "": + print(f"The following changes have been added to this branch in the library:\n\n{lgLib}\n\n") + print(f"ERROR: You must push these changes before you can pull or reset this branch to {remoteCommitId}") + sys.exit(1) + + # get log with relevant commits in this repo that are to be pulled + lg = self.gitLogThisRepoSinceLastSync() + + os.chdir(libRepo.rootPath) + + # create commit message file + commitMsg = f"Sync {self.name}\n\n" + lg + with open(COMMIT_MSG_FILENAME, "w") as f: + f.write(commitMsg) + + # ask whether to commit these changes + print("Relevant commits:\n\n" + lg + "\n\n") + if not self._userInputYesNo(f"The above changes will be pulled from {self.name}.\n" + f"You may change the commit message by editing {os.path.abspath(COMMIT_MSG_FILENAME)}.\n" + "Continue?"): + os.unlink(COMMIT_MSG_FILENAME) + return + + # prepare restoration of ignored files + self.prepare_restoration_of_ignored_files(libRepo.rootPath) + + # remove library tree in lib repo + shutil.rmtree(self.libRepo.libDirectory) + + # copy tree from this repo to lib repo (but drop the sync commit id files) + shutil.copytree(self.pathToLibInThisRepo, self.libRepo.libDirectory) + for fn in (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO): + p = os.path.join(self.libRepo.libDirectory, fn) + if os.path.exists(p): + os.unlink(p) + + # restore ignored directories/files + self.restore_ignored_files(libRepo.rootPath) + + # make commit in lib repo + os.system("git add %s" % self.libRepo.libDirectory) + gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME) + newSyncCommitIdLibRepo = call("git rev-parse HEAD").strip() + + # update commit ids in this repo + os.chdir(self.pathToLibInThisRepo) + newSyncCommitIdThisRepo = call("git rev-parse HEAD").strip() + with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f: + f.write(newSyncCommitIdLibRepo) + with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f: + f.write(newSyncCommitIdThisRepo) + execute('git add %s %s' % (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO)) + execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (pull)"') + + print(f"\n\nIf everything was successful, you should now push your changes to branch " + f"'{self.branch}'\nand get your branch merged into develop (issuing a pull request where appropriate)") + + def push(self, libRepo: "LibRepo"): + """ + Pushes changes from the lib repo to this repo + """ + os.chdir(libRepo.rootPath) + + # switch to the source repo branch + execute(f"git checkout {self.branch}") + + if self.isSyncEstablished(): + + # check if there are any commits that have not yet been pulled + unpulledCommits = self.gitLogThisRepoSinceLastSync().strip() + if unpulledCommits != "": + print(f"\n{unpulledCommits}\n\n") + if not self._userInputYesNo(f"WARNING: The above changes in repository '{self.name}' have not" + f" yet been pulled.\nYou might want to pull them.\n" + f"If you continue with the push, they will be lost. Continue?"): + return + + # get change log in lib repo since last sync + libLogSinceLastSync = self.gitLogLibRepoSinceLastSync(libRepo) + + print("Relevant commits:\n\n" + libLogSinceLastSync + "\n\n") + if not self._userInputYesNo("The above changes will be pushed. Continue?"): + return + print() + else: + libLogSinceLastSync = "" + + # prepare restoration of ignored files in target repo + base_dir_this_repo = os.path.join(self.pathToLibInThisRepo, "..") + self.prepare_restoration_of_ignored_files(base_dir_this_repo) + + # remove the target repo tree and update it with the tree from the source repo + shutil.rmtree(self.pathToLibInThisRepo) + shutil.copytree(libRepo.libPath, self.pathToLibInThisRepo) + + # get the commit id of the source repo we just copied + commitId = call("git rev-parse HEAD").strip() + + # restore ignored directories and files + self.restore_ignored_files(base_dir_this_repo) + + # go to the target repo + os.chdir(self.pathToLibInThisRepo) + + # commit new version in this repo + execute("git add .") + with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f: + f.write(commitId) + execute("git add %s" % self.SYNC_COMMIT_ID_FILE_LIB_REPO) + gitCommit(f"{self.libRepo.name} {commitId}" + libLogSinceLastSync) + commitId = call("git rev-parse HEAD").strip() + + # update information on the commit id we just added + with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f: + f.write(commitId) + execute("git add %s" % self.SYNC_COMMIT_ID_FILE_THIS_REPO) + execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (push)"') + + os.chdir(libRepo.rootPath) + + print(f"\n\nIf everything was successful, you should now update the remote branch:\ngit push") + + def prepare_restoration_of_ignored_files(self, base_dir: str): + """ + :param base_dir: the directory containing the lib directory, to which ignored paths are relative + """ + cwd = os.getcwd() + os.chdir(base_dir) + + # ensure backup dir exists and is empty + if os.path.exists(self.SYNC_BACKUP_DIR): + shutil.rmtree(self.SYNC_BACKUP_DIR) + os.mkdir(self.SYNC_BACKUP_DIR) + + # backup ignored, unversioned directories + for d in self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(d): + shutil.copytree(d, os.path.join(self.SYNC_BACKUP_DIR, d)) + + os.chdir(cwd) + + def restore_ignored_files(self, base_dir: str): + """ + :param base_dir: the directory containing the lib directory, to which ignored paths are relative + """ + cwd = os.getcwd() + os.chdir(base_dir) + + # remove fully ignored directories that were overwritten by the sync + for d in self.libRepo.fullyIgnoredVersionedDirectories + self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(d): + print("Removing overwritten content: %s" % d) + shutil.rmtree(d) + + # restore directories and files that can be restored via git + for d in self.libRepo.ignoredDirectories + self.libRepo.fullyIgnoredVersionedDirectories: + restoration_cmd = "git checkout %s" % d + print("Restoring: %s" % restoration_cmd) + os.system(restoration_cmd) + for pattern in self.libRepo.ignoredFileGlobPatterns: + for path in glob.glob(pattern, recursive=True): + print("Restoring via git: %s" % path) + os.system("git checkout %s" % path) + + # restore directories that were backed up + for d in self.libRepo.fullyIgnoredUnversionedDirectories: + if os.path.exists(os.path.join(self.SYNC_BACKUP_DIR, d)): + print("Restoring from backup: %s" % d) + shutil.copytree(os.path.join(self.SYNC_BACKUP_DIR, d), d) + + # remove backup dir + shutil.rmtree(self.SYNC_BACKUP_DIR) + + os.chdir(cwd) + + +class LibRepo: + def __init__(self, name: str, libDirectory: str, + ignoredDirectories: Sequence[str] = (), + fullyIgnoredVersionedDirectories: Sequence[str] = (), + fullyIgnoredUnversionedDirectories: Sequence[str] = (), + ignoredFileGlobPatterns: Sequence[str] = () + ): + """ + :param name: name of the library being synced + :param libDirectory: relative path to the library directory within this repo + :param ignoredDirectories: ignored directories; existing files in ignored directories will be restored + via 'git checkout' on pull/push, but new files will be added. + This is useful for configuration-like files, where users may have local changes that should not + be overwritten, but new files should still be added. + :param fullyIgnoredVersionedDirectories: + fully ignored versioned directories will be restored to original state after push/pull via git checkout + :param fullyIgnoredUnversionedDirectories: + fully ignored unversioned directories will be backed up and restored to original state after push/pull + :param ignoredFileGlobPatterns: files matching ignored glob patterns will be restored via 'git checkout' + on pull/push + """ + self.name = name + self.rootPath = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + self.libDirectory = libDirectory + self.libPath = os.path.join(self.rootPath, self.libDirectory) + self.ignoredDirectories: List[str] = list(ignoredDirectories) + self.fullyIgnoredVersionedDirectories: List[str] = list(fullyIgnoredVersionedDirectories) + self.fullyIgnoredUnversionedDirectories: List[str] = list(fullyIgnoredUnversionedDirectories) + self.ignoredFileGlobPatterns: List[str] = list(ignoredFileGlobPatterns) + self.otherRepos: List[OtherRepo] = [] + + def add(self, repo: OtherRepo): + repo.libRepo = self + self.otherRepos.append(repo) + + def runMain(self): + repos = self.otherRepos + args = sys.argv[1:] + if len(args) != 2: + print(f"usage: sync.py <{'|'.join([repo.name for repo in repos])}> ") + else: + repo = [r for r in repos if r.name == args[0]] + if len(repo) != 1: + raise ValueError(f"Unknown repo '{args[0]}'") + repo = repo[0] + + if args[1] == "push": + repo.push(self) + elif args[1] == "pull": + repo.pull(self) + else: + raise ValueError(f"Unknown command '{args[1]}'") diff --git a/sync.py b/sync.py new file mode 100644 index 00000000..08cb8d5d --- /dev/null +++ b/sync.py @@ -0,0 +1,6 @@ +import os +from repo_dir_sync import LibRepo, OtherRepo + +r = LibRepo(name="serena", libDirectory="src") +r.add(OtherRepo(name="mux", branch="mux", pathToLib=os.path.join("..", "serena-multiplexer", "src-serena"))) +r.runMain()