pelican-deployment-system/pelican_deploy/deploy.py

   Copyright 2016 Peter Dahlberg

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

from pathlib import Path
from collections import namedtuple, deque
from pelican_deploy.gittool import Repo, log_git_result
from functools import partial
from subprocess import Popen, PIPE
from pelican_deploy.util import exception_logged
from concurrent.futures import ThreadPoolExecutor
from threading import RLock
from datetime import datetime
import pytz
import sys
import logging
import shlex
import os

log = logging.getLogger(__name__)

log_git = partial(log_git_result, out_logger=log.debug,
                  err_logger=log.debug, status_logger=log.debug)

TOX_RESULT_FILE = "{name}_result.json"
BUILD_REPO_DIR = "{name}_build_repo"
OUTPUT_DIR = "{name}_output"
STATUS_LEN = 500

BuildStatus = namedtuple("BuildStatus", "date ok msg payload running")

class PullError(Exception):
    pass

class DeploymentRunner:

    def __init__(self, name, runner_config):
        self.name = name
        self.working_directory = Path(runner_config["working_directory"])
        if not self.working_directory.exists():
            log.info("creating working directory for %s: %s", name,
                     self.working_directory)
            self.working_directory.mkdir(parents=True)
        self.working_directory = self.working_directory.resolve()

        self.clone_url = runner_config["clone_url"]
        self.git_branch = runner_config["git_branch"]
        self.build_repo_path = self.working_directory / BUILD_REPO_DIR.format(
            name=name)
        outdir = self.working_directory / OUTPUT_DIR.format(name=name)
        toxresult = self.working_directory / TOX_RESULT_FILE.format(name=name)
        self.build_command = runner_config["build_command"].format(
            output=outdir, toxresult=toxresult)
        self.final_install_command = runner_config["final_install_command"]\
            .format(output=outdir)

        self._build_proc_env = dict(os.environ,
                                    **runner_config.get("build_env", {}))

        self._executor = ThreadPoolExecutor(max_workers=1)
        self._futures = set()
        self._build_proc = None
        self._abort = False
        self._build_lock = RLock()
        self._repo_update_lock = RLock()

        self.build_status = deque(maxlen=STATUS_LEN)

    def update_status(self, ok, msg, payload=None, running=True):
        date = pytz.utc.localize(datetime.utcnow())
        self.build_status.append(BuildStatus(date, ok, msg, payload, running))

    def update_build_repository(self):
        with self._repo_update_lock:
            self._update_build_repository()

    def _update_build_repository(self):
        if not self.build_repo_path.exists():
            self.build_repo_path.mkdir(parents=True)

        repo = Repo(str(self.build_repo_path))
        if not repo.is_repo():
            if self.build_repo_path.is_dir() and \
                    next(self.build_repo_path.iterdir(), None) is not None:
                log.error(
                    "non-empty %s exists but not a valid git repository!",
                    self.build_repo_path)
                raise RuntimeException(("non-empty {} exists but not a"
                    "valid git repository!").format(self.build_repo_path))
            else:
                log.info("Build repository %s not there, cloning",
                         self.build_repo_path)
                result = repo.clone("--branch", self.git_branch,
                                    "--depth", "1", self.clone_url, ".")
                log_git(result)


        origin_url = repo.config_get("remote.origin.url")
        if origin_url != self.clone_url:
            log.info("%s build_repo: URL of git origin changed (`%s` --> `%s`),\
                     adjusting...", self.name, origin_url, self.clone_url)
            repo.config("remote.origin.url", self.clone_url)

        # deinit submodules to avoid removed ones dangling around later
        # they should stay around in .git, so reinit should be fast
        result = repo.submodule("deinit", "--force", ".")
        log_git(result)

        result = repo.checkout("--force", self.git_branch)
        log_git(result)

        result = repo.reset("--hard")
        log_git(result)

        log.info("%s build_repo: pulling changes from origin", self.name)
        refspec = "+{b}:{b}".format(b=self.git_branch)
        try:
            result = repo.pull("--force", "--recurse-submodules",
                               "--depth", "1", "origin", refspec)
            log_git(result)
        except Exception as e:
            # need to reinit the submodules
            self._update_build_repo_submodules(repo)
            raise PullError from e

        try:
            result = repo.clean("--force", "-d", "-x")
            log_git(result)
        except:
            log.warning("git clean failed!", exc_info=True)

        # update the submodules
        self._update_build_repo_submodules(repo)

    def _update_build_repo_submodules(self, repo):
        log.info("%s build_repo: update submodules", self.name)
        result = repo.submodule("update", "--init", "--force", "--recursive")
        log_git(result)

    def build(self, abort_running=False, wait=False, ignore_pull_error=False):
        with self._build_lock:
            if abort_running:
                self.try_abort_build()

            # cancel everything, so we are next
            for fut in self._futures.copy():
                fut.cancel()
                if fut.done():
                    self._futures.remove(fut)

            def build_job():
                build_bl = partial(self._build_blocking, ignore_pull_error=
                                   ignore_pull_error)
                build_func = exception_logged(build_bl, log.error)
                try:
                    build_func()
                except Exception as e:
                    self.update_status(False, "Build stopped with exception",
                                       running=False, payload={"exception": e})
                    raise

            future = self._executor.submit(build_job)
            self._futures.add(future)
        if wait:
            return future.result()

    def try_abort_build(self):
        proc = self._build_proc
        self._abort = True
        if proc:
            try:
                proc.kill()
            except:
                log.debug("unable to kill", exc_info=True)

    def final_install(self):
        args = shlex.split(self.final_install_command)
        self.update_status(True, "Starting final_install",
                           payload={"cmd": args})
        log.info("%s: Starting final_install `%s`", self.name, args)
        proc = Popen(args, stdout=PIPE, stderr=PIPE, universal_newlines=True,
                     start_new_session=True)
        outs, errs = proc.communicate()
        status = proc.wait()

        if status < 0:
            log.info("%s: killed final_install_command (%s)", self.name, status)
        else:
            log.info("%s: finished final_install_command with status %s!",
                        self.name, status)
            log.info('%s final_install_command stdout: %s\n', self.name, outs)
            log.info('%s final_install_command stderr: %s\n', self.name, errs)

        if status > 0:
            self.update_status(False, ("final_install_command failed."
                               " Website may be broken!"),
                               payload={"status": status,
                                        "stdout": outs, "stderr": errs})
            log.error("%s: final_install failed! Website may be broken!",
                      self.name)
        else:
            self.update_status(True, "finished final_install_command",
                               payload={"stdout": outs, "stderr": errs})

    def _build_blocking(self, ignore_pull_error=False):
        self._abort = False

        # preparing build environment
        try:
            self.update_status(True, "Start updating repository")
            self.update_build_repository()
        except PullError:
            if ignore_pull_error:
                msg = "Git pull failed, trying to continue with what we have"
                self.update_status(False, msg)
                log.warning(msg, exc_info=True)
            else:
                raise

        # start the build if we should not abort
        if not self._abort:
            args = shlex.split(self.build_command)
            self.update_status(True, "Starting the main build command",
                               payload={"cmd": args})
            log.info("%s: Starting build_command `%s`", self.name, args)
            self._build_proc = Popen(args, stdout=PIPE, stderr=PIPE,
                                     cwd=str(self.build_repo_path),
                                     env=self._build_proc_env,
                                     universal_newlines=True,
                                     start_new_session=True)
            outs, errs = self._build_proc.communicate()
            status = self._build_proc.wait()
            self._build_proc = None

            if status < 0:
                self.update_status(False, "killed build_command")
                log.info("%s: killed build_command", self.name)
            else:
                log.info("%s: finished build_command with status %s!",
                         self.name, status)
                log.info('%s build_command stdout: %s\n', self.name, outs)
                log.info('%s build_command stderr: %s\n', self.name, errs)
            if status == 0:
                self.update_status(True, "finished build_command",
                                   payload={"stdout": outs, "stderr": errs})
                self.final_install()
            else:
                self.update_status(False, "build_command failed",
                                   payload={"status": status,
                                   "stdout": outs, "stderr": errs})

            self.update_status(self.build_status[-1].ok, "End of build",
                               running=False)


    def shutdown(self):
        self.try_abort_build()
        self._executor.shutdown(wait=True)