Copyright 2016 Peter Dahlberg Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. from pathlib import Path from collections import namedtuple, deque from pelican_deploy.gittool import Repo, log_git_result from functools import partial from subprocess import Popen, PIPE from pelican_deploy.util import exception_logged from concurrent.futures import ThreadPoolExecutor from threading import RLock from datetime import datetime import pytz import sys import logging import shlex import os log = logging.getLogger(__name__) log_git = partial(log_git_result, out_logger=log.debug, err_logger=log.debug, status_logger=log.debug) TOX_RESULT_FILE = "{name}_result.json" BUILD_REPO_DIR = "{name}_build_repo" OUTPUT_DIR = "{name}_output" STATUS_LEN = 500 BuildStatus = namedtuple("BuildStatus", "date ok msg payload running") class PullError(Exception): pass class DeploymentRunner: def __init__(self, name, runner_config): self.name = name self.working_directory = Path(runner_config["working_directory"]) if not self.working_directory.exists(): log.info("creating working directory for %s: %s", name, self.working_directory) self.working_directory.mkdir(parents=True) self.working_directory = self.working_directory.resolve() self.clone_url = runner_config["clone_url"] self.git_branch = runner_config["git_branch"] self.build_repo_path = self.working_directory / BUILD_REPO_DIR.format( name=name) outdir = self.working_directory / OUTPUT_DIR.format(name=name) toxresult = self.working_directory / TOX_RESULT_FILE.format(name=name) self.build_command = runner_config["build_command"].format( output=outdir, toxresult=toxresult) self.final_install_command = runner_config["final_install_command"]\ .format(output=outdir) self._build_proc_env = dict(os.environ, **runner_config.get("build_env", {})) self._executor = ThreadPoolExecutor(max_workers=1) self._futures = set() self._build_proc = None self._abort = False self._build_lock = RLock() self._repo_update_lock = RLock() self.build_status = deque(maxlen=STATUS_LEN) def update_status(self, ok, msg, payload=None, running=True): date = pytz.utc.localize(datetime.utcnow()) self.build_status.append(BuildStatus(date, ok, msg, payload, running)) def update_build_repository(self): with self._repo_update_lock: self._update_build_repository() def _update_build_repository(self): if not self.build_repo_path.exists(): self.build_repo_path.mkdir(parents=True) repo = Repo(str(self.build_repo_path)) if not repo.is_repo(): if self.build_repo_path.is_dir() and \ next(self.build_repo_path.iterdir(), None) is not None: log.error( "non-empty %s exists but not a valid git repository!", self.build_repo_path) raise RuntimeException(("non-empty {} exists but not a" "valid git repository!").format(self.build_repo_path)) else: log.info("Build repository %s not there, cloning", self.build_repo_path) result = repo.clone("--branch", self.git_branch, "--depth", "1", self.clone_url, ".") log_git(result) origin_url = repo.config_get("remote.origin.url") if origin_url != self.clone_url: log.info("%s build_repo: URL of git origin changed (`%s` --> `%s`),\ adjusting...", self.name, origin_url, self.clone_url) repo.config("remote.origin.url", self.clone_url) # deinit submodules to avoid removed ones dangling around later # they should stay around in .git, so reinit should be fast result = repo.submodule("deinit", "--force", ".") log_git(result) result = repo.checkout("--force", self.git_branch) log_git(result) result = repo.reset("--hard") log_git(result) log.info("%s build_repo: pulling changes from origin", self.name) refspec = "+{b}:{b}".format(b=self.git_branch) try: result = repo.pull("--force", "--recurse-submodules", "--depth", "1", "origin", refspec) log_git(result) except Exception as e: # need to reinit the submodules self._update_build_repo_submodules(repo) raise PullError from e try: result = repo.clean("--force", "-d", "-x") log_git(result) except: log.warning("git clean failed!", exc_info=True) # update the submodules self._update_build_repo_submodules(repo) def _update_build_repo_submodules(self, repo): log.info("%s build_repo: update submodules", self.name) result = repo.submodule("update", "--init", "--force", "--recursive") log_git(result) def build(self, abort_running=False, wait=False, ignore_pull_error=False): with self._build_lock: if abort_running: self.try_abort_build() # cancel everything, so we are next for fut in self._futures.copy(): fut.cancel() if fut.done(): self._futures.remove(fut) def build_job(): build_bl = partial(self._build_blocking, ignore_pull_error= ignore_pull_error) build_func = exception_logged(build_bl, log.error) try: build_func() except Exception as e: self.update_status(False, "Build stopped with exception", running=False, payload={"exception": e}) raise future = self._executor.submit(build_job) self._futures.add(future) if wait: return future.result() def try_abort_build(self): proc = self._build_proc self._abort = True if proc: try: proc.kill() except: log.debug("unable to kill", exc_info=True) def final_install(self): args = shlex.split(self.final_install_command) self.update_status(True, "Starting final_install", payload={"cmd": args}) log.info("%s: Starting final_install `%s`", self.name, args) proc = Popen(args, stdout=PIPE, stderr=PIPE, universal_newlines=True, start_new_session=True) outs, errs = proc.communicate() status = proc.wait() if status < 0: log.info("%s: killed final_install_command (%s)", self.name, status) else: log.info("%s: finished final_install_command with status %s!", self.name, status) log.info('%s final_install_command stdout: %s\n', self.name, outs) log.info('%s final_install_command stderr: %s\n', self.name, errs) if status > 0: self.update_status(False, ("final_install_command failed." " Website may be broken!"), payload={"status": status, "stdout": outs, "stderr": errs}) log.error("%s: final_install failed! Website may be broken!", self.name) else: self.update_status(True, "finished final_install_command", payload={"stdout": outs, "stderr": errs}) def _build_blocking(self, ignore_pull_error=False): self._abort = False # preparing build environment try: self.update_status(True, "Start updating repository") self.update_build_repository() except PullError: if ignore_pull_error: msg = "Git pull failed, trying to continue with what we have" self.update_status(False, msg) log.warning(msg, exc_info=True) else: raise # start the build if we should not abort if not self._abort: args = shlex.split(self.build_command) self.update_status(True, "Starting the main build command", payload={"cmd": args}) log.info("%s: Starting build_command `%s`", self.name, args) self._build_proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=str(self.build_repo_path), env=self._build_proc_env, universal_newlines=True, start_new_session=True) outs, errs = self._build_proc.communicate() status = self._build_proc.wait() self._build_proc = None if status < 0: self.update_status(False, "killed build_command") log.info("%s: killed build_command", self.name) else: log.info("%s: finished build_command with status %s!", self.name, status) log.info('%s build_command stdout: %s\n', self.name, outs) log.info('%s build_command stderr: %s\n', self.name, errs) if status == 0: self.update_status(True, "finished build_command", payload={"stdout": outs, "stderr": errs}) self.final_install() else: self.update_status(False, "build_command failed", payload={"status": status, "stdout": outs, "stderr": errs}) self.update_status(self.build_status[-1].ok, "End of build", running=False) def shutdown(self): self.try_abort_build() self._executor.shutdown(wait=True)