Source code for pydmt.core.pydmt

"""
pydmt.py
"""

import logging

import os

from pydmt.api.builder import Builder
from pydmt.core.cache import Cache
from pydmt.utils.filesystem import copy_mkdir
from pydmt.utils.digest import sha1_file
from pydmt.configs import ConfigFlow
from pydmt.static import LOGGER_NAME


[docs] class BuildProcessStats: def __init__(self): self.builder_fail = [] self.builder_ok = [] self.copy_missing = [] self.copy_sha1 = [] self.nop = []
[docs] def add_copy_missing(self, filename: str, object_name: str): self.copy_missing.append((filename, object_name))
[docs] def get_copy_missing(self): return len(self.copy_missing)
[docs] def add_copy_sha1(self, filename: str, object_name: str): self.copy_sha1.append((filename, object_name))
[docs] def add_nop(self, filename: str, object_name: str): self.nop.append((filename, object_name))
[docs] def get_nop(self): return len(self.nop)
[docs] def add_builder_ok(self, builder: Builder): self.builder_ok.append(builder)
[docs] def add_builder_fail(self, builder: Builder, e: Exception): self.builder_fail.append((builder, e))
[docs] def get_builder_ok(self): return len(self.builder_ok)
[docs] def get_builder_fail(self): return len(self.builder_fail)
[docs] def get_os_error_code(self) -> int: if len(self.builder_fail) > 0: return 1 return 0
[docs] class PyDMT: def __init__(self): self.builders: list[Builder] = [] self.target_to_builder: dict[str, Builder] = {} self.cache = Cache()
[docs] def build_by_builder(self, builder: Builder, stats: BuildProcessStats): """ run one builder, return statistics about the run """ logger = logging.getLogger(LOGGER_NAME) target_signature = builder.get_signature() logger.debug(f"examining [{builder.get_name()}]") if self.cache.list_sig_ok(target_signature): logger.debug(f"verifying [{builder.get_name()}]") file_bad = 0 file_correct = 0 file_missing = 0 file_total = 0 list_filename = self.cache.get_list_filename(target_signature) for object_name, signature in Cache.iterate_objects(list_filename): filename = self.cache.get_object_filename(signature) if os.path.isfile(object_name): object_name_signature = sha1_file(object_name) if object_name_signature != signature: logger.debug(f"file [{object_name}] is incorrect. Getting from cache.") copy_mkdir(filename, object_name) stats.add_copy_sha1(filename, object_name) file_bad += 1 else: logger.debug(f"file [{object_name}] is up to date") stats.add_nop(filename, object_name) file_correct += 1 else: logger.debug(f"file [{object_name}] is missing. Getting from cache.") copy_mkdir(filename, object_name) stats.add_copy_missing(filename, object_name) file_missing += 1 file_total += 1 if file_bad > 0 or file_missing > 0: logger.debug(f"Retrieved {file_total} files from cache") logger.debug(f"(bad/correct/missing = {file_bad}/{file_correct}/{file_missing}") else: logger.debug(f"ok [{builder.get_name()}]") else: logger.debug(f"running [{builder.get_name()}]") print(f"{builder.get_name()}: [{builder.get_targets_as_string()}]...", end="", flush=True) # this is one of the rare cases in which we really want to catch all exceptions. # pylint: disable=broad-except # noinspection PyBroadException try: builder.build() except Exception as e: print("FAIL") logger.exception("exception") stats.add_builder_fail(builder, e) return False print("OK") stats.add_builder_ok(builder) d = {} for signature, target in builder.yield_results(): self.cache.save_object_by_signature(signature, target) d[target] = signature self.cache.save_list_by_signature(target_signature, d) return True
[docs] def build_by_target(self, target: str, stats: BuildProcessStats) -> None: b = self.target_to_builder[target] self.build_by_builder(b, stats)
[docs] def build_by_targets(self, targets: list[str], stats: BuildProcessStats) -> None: for target in targets: self.build_by_target(target, stats)
[docs] def build_all(self) -> BuildProcessStats: """ Build all the targets, very high level method TODO: order builders by depenencies and do this multi-core :return: statistics about the build """ stats = BuildProcessStats() for builder in self.builders: res = self.build_by_builder( builder=builder, stats=stats, ) if not res and ConfigFlow.stop_after_error: return stats return stats
[docs] def clean_all(self) -> None: """ Clean all targets """ for builder in self.builders: for target in builder.get_targets(): target.remove()
[docs] def add_builder(self, b: Builder) -> None: self.builders.append(b) for target in b.get_targets(): self.target_to_builder[target.get_name()] = b