Source code for pydmt.core.pydmt
"""
pydmt.py
"""
import logging
import os
from pydmt.api.builder import Builder
from pydmt.core.cache import Cache
from pydmt.utils.filesystem import copy_mkdir
from pydmt.utils.digest import sha1_file
from pydmt.configs import ConfigFlow
from pydmt.static import LOGGER_NAME
[docs]
class BuildProcessStats:
def __init__(self):
self.builder_fail = []
self.builder_ok = []
self.copy_missing = []
self.copy_sha1 = []
self.nop = []
[docs]
def add_copy_missing(self, filename: str, object_name: str):
self.copy_missing.append((filename, object_name))
[docs]
def get_copy_missing(self):
return len(self.copy_missing)
[docs]
def add_copy_sha1(self, filename: str, object_name: str):
self.copy_sha1.append((filename, object_name))
[docs]
def add_nop(self, filename: str, object_name: str):
self.nop.append((filename, object_name))
[docs]
def get_nop(self):
return len(self.nop)
[docs]
def add_builder_ok(self, builder: Builder):
self.builder_ok.append(builder)
[docs]
def add_builder_fail(self, builder: Builder, e: Exception):
self.builder_fail.append((builder, e))
[docs]
def get_builder_ok(self):
return len(self.builder_ok)
[docs]
def get_builder_fail(self):
return len(self.builder_fail)
[docs]
def get_os_error_code(self) -> int:
if len(self.builder_fail) > 0:
return 1
return 0
[docs]
class PyDMT:
def __init__(self):
self.builders: list[Builder] = []
self.target_to_builder: dict[str, Builder] = {}
self.cache = Cache()
[docs]
def build_by_builder(self, builder: Builder, stats: BuildProcessStats):
""" run one builder, return statistics about the run """
logger = logging.getLogger(LOGGER_NAME)
target_signature = builder.get_signature()
logger.debug(f"examining [{builder.get_name()}]")
if self.cache.list_sig_ok(target_signature):
logger.debug(f"verifying [{builder.get_name()}]")
file_bad = 0
file_correct = 0
file_missing = 0
file_total = 0
list_filename = self.cache.get_list_filename(target_signature)
for object_name, signature in Cache.iterate_objects(list_filename):
filename = self.cache.get_object_filename(signature)
if os.path.isfile(object_name):
object_name_signature = sha1_file(object_name)
if object_name_signature != signature:
logger.debug(f"file [{object_name}] is incorrect. Getting from cache.")
copy_mkdir(filename, object_name)
stats.add_copy_sha1(filename, object_name)
file_bad += 1
else:
logger.debug(f"file [{object_name}] is up to date")
stats.add_nop(filename, object_name)
file_correct += 1
else:
logger.debug(f"file [{object_name}] is missing. Getting from cache.")
copy_mkdir(filename, object_name)
stats.add_copy_missing(filename, object_name)
file_missing += 1
file_total += 1
if file_bad > 0 or file_missing > 0:
logger.debug(f"Retrieved {file_total} files from cache")
logger.debug(f"(bad/correct/missing = {file_bad}/{file_correct}/{file_missing}")
else:
logger.debug(f"ok [{builder.get_name()}]")
else:
logger.debug(f"running [{builder.get_name()}]")
print(f"{builder.get_name()}: [{builder.get_targets_as_string()}]...", end="", flush=True)
# this is one of the rare cases in which we really want to catch all exceptions.
# pylint: disable=broad-except
# noinspection PyBroadException
try:
builder.build()
except Exception as e:
print("FAIL")
logger.exception("exception")
stats.add_builder_fail(builder, e)
return False
print("OK")
stats.add_builder_ok(builder)
d = {}
for signature, target in builder.yield_results():
self.cache.save_object_by_signature(signature, target)
d[target] = signature
self.cache.save_list_by_signature(target_signature, d)
return True
[docs]
def build_by_target(self, target: str, stats: BuildProcessStats) -> None:
b = self.target_to_builder[target]
self.build_by_builder(b, stats)
[docs]
def build_by_targets(self, targets: list[str], stats: BuildProcessStats) -> None:
for target in targets:
self.build_by_target(target, stats)
[docs]
def build_all(self) -> BuildProcessStats:
"""
Build all the targets, very high level method
TODO: order builders by depenencies and do this multi-core
:return: statistics about the build
"""
stats = BuildProcessStats()
for builder in self.builders:
res = self.build_by_builder(
builder=builder,
stats=stats,
)
if not res and ConfigFlow.stop_after_error:
return stats
return stats
[docs]
def clean_all(self) -> None:
"""
Clean all targets
"""
for builder in self.builders:
for target in builder.get_targets():
target.remove()
[docs]
def add_builder(self, b: Builder) -> None:
self.builders.append(b)
for target in b.get_targets():
self.target_to_builder[target.get_name()] = b