rework submodule updating, they suck really!

pelican_deploy/deploy.py

@@ -144,11 +144,9 @@ class DeploymentRunner:
 
     def _update_build_repo_submodules(self, repo):
         log.info("%s build_repo: update submodules", self.name)
-        # we must update the urls if changed!
-        result = repo.submodule("sync", "--recursive")
-        log_git(result)
-        result = repo.submodule("update", "--init", "--force", "--recursive")
-        log_git(result)
+        results = repo.submodule_sync_update_init_recursive_force()
+        for r in results:
+            log_git(r)
 
     def build(self, abort_running=False, wait=False, ignore_pull_error=False):
         with self._build_lock:

pelican_deploy/gittool.py

@@ -15,7 +15,7 @@
 import os
 import errno
 import shlex
-from collections import namedtuple
+from collections import namedtuple, deque
 from subprocess import Popen, PIPE
 
 CmdResult = namedtuple("CmdResult", "cmd status stdout stderr")
@@ -69,6 +69,33 @@ class Repo:
         res = self.config("--get", key)
         return res.stdout.rstrip("\r\n")
 
+    def submodule_sync_update_init_recursive_force(self):
+        results = []
+        todo = deque()
+        todo.extend(self._get_submod_paths())
+        while todo:
+            curr = todo.popleft()
+            if not os.path.exists(os.path.join(self.repo_dir, curr)):
+                continue #  that happens, strangely...
+            results.append(self.cmd(self.git_cmd, "-C", curr,
+                                    "submodule", "sync"))
+            results.append(self.cmd(self.git_cmd, "-C", curr,
+                                    "submodule", "update", "--init", "--force"))
+            todo.extend(os.path.join(curr, p) for p in
+                        self._get_submod_paths(curr))
+        return results
+
+    def _get_submod_paths(self, submod="."):
+        if not os.path.exists(os.path.join(self.repo_dir, submod,
+                                           ".gitmodules")):
+            return ()
+
+        result = self.cmd(self.git_cmd, "-C", submod, "config", "--file",
+                          ".gitmodules", "--get-regexp", "submodule\..*\.path")
+        return tuple(p.split(maxsplit=1)[1] for p in result.stdout.splitlines())
+
+
+
 def log_git_result(result, out_logger=None, err_logger=None, status_logger=None):
     if status_logger:
         err_logger('%s exit status: %s', result.cmd, result.status)