Coverage for gidappdata\cli\skeleton_tree.py : 78%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# region [Imports]
3# * Standard Library Imports -->
4import gc
5import os
6import re
7import sys
8import json
9import lzma
10import time
11import queue
12import logging
13import platform
14import subprocess
15from enum import Enum, Flag, auto
16from time import sleep
17from pprint import pprint, pformat
18from typing import Union
19from datetime import tzinfo, datetime, timezone, timedelta
20from functools import wraps, lru_cache, singledispatch, total_ordering, partial
21from contextlib import contextmanager
22from collections import Counter, ChainMap, deque, namedtuple, defaultdict
23from multiprocessing import Pool
24from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
26# * Third Party Imports -->
27# import requests
28# import pyperclip
29# import matplotlib.pyplot as plt
30# from bs4 import BeautifulSoup
31# from dotenv import load_dotenv
32# from github import Github, GithubException
33# from jinja2 import BaseLoader, Environment
34# from natsort import natsorted
35# from fuzzywuzzy import fuzz, process
37# * Gid Imports -->
38import gidlogger as glog
40from gidappdata.utility import readit, readbin, writebin, writeit, writejson, loadjson, linereadit, pathmaker, create_folder, create_file, clearit, pickleit, get_pickled, read_file
41import base64
43# endregion[Imports]
46# region [AppUserData]
48# endregion [AppUserData]
50# region [Logging]
52log = logging.getLogger('gidappdata')
53log.info(glog.imported(__name__))
55# endregion[Logging]
57THIS_FILE_DIR = os.path.abspath(os.path.dirname(__file__))
60def json_parts(item):
61 if item.typus is not SkeletonTypus.Root:
62 yield (item.parent.name, item.name, item.typus.value, item.content)
64 if item.typus is not SkeletonTypus.File:
65 for child in item.children:
66 yield from json_parts(child)
69def json_it(item, _file):
70 _out = {}
71 for _parent, _name, _typus, _content in json_parts(item):
72 if _parent not in _out:
73 _out[_parent] = []
74 if isinstance(_content, bytes):
75 _content = base64.b64encode(_content)
76 _out[_parent].append((_name, _typus, _content))
77 writejson(_out, _file, indent=2, sort_keys=False)
80class SkeletonTypus(Enum):
81 Folder = 'folder'
82 File = 'file'
83 Root = 'root'
86class SkeletonInstructionBaseItem:
87 pass
90class SkeletonInstructionItem(SkeletonInstructionBaseItem):
91 serialize_strategies = {'pickle': (pickleit, '.pkl'), 'json': (json_it, '.json')}
92 standard_root_name = 'data_pack'
94 def __init__(self, name: str, typus: SkeletonTypus, parent: SkeletonInstructionBaseItem = None, content=None):
95 self.name = name
96 self.typus = typus
97 self.parent = parent
98 self.children = [] if self.typus in [SkeletonTypus.Folder, SkeletonTypus.Root] else None
99 self.content = content
101 @staticmethod
102 def _make_child_keyname(child):
103 return child.name.replace('.', '__')
105 def add_child_item(self, new_child: SkeletonInstructionBaseItem):
106 if self.typus is SkeletonTypus.File:
107 raise AttributeError("Files can not have children Items")
108 if any(new_child.name.casefold() == existing_child.name.casefold() for existing_child in self.children):
109 raise FileExistsError(f"The {new_child.typus.value} {new_child.name} already exist inside this Folder")
110 self.children.append(new_child)
111 new_child.parent = self
113 @property
114 def path(self):
115 if self.parent is None or self.typus is SkeletonTypus.Root:
116 return pathmaker(self.name)
117 else:
118 return pathmaker(self.parent.path, self.name)
120 def set_root_path(self, root_path):
121 if self.typus is SkeletonTypus.Root:
122 self.name = root_path
123 elif self.parent is None:
124 raise AttributeError("This method can not be used on items that are not linked utimately to an ROOT Item")
125 else:
126 self.parent.set_root_path(root_path)
128 def get_paths(self):
129 yield self.path
130 if self.typus is not SkeletonTypus.File:
131 for child in self.children:
132 yield from child.get_paths()
134 def _build(self, overwrite=False):
135 if self.typus is SkeletonTypus.Root:
136 create_folder(self.path, False)
137 elif self.typus is SkeletonTypus.Folder:
138 create_folder(self.path, overwrite=overwrite)
140 elif self.typus is SkeletonTypus.File:
141 content = '' if self.content is None else self.content
142 create_file(self.path, content, overwrite=overwrite)
144 if self.children is not None:
145 for child in self.children:
146 child._build(overwrite=overwrite)
148 def start_build(self, overwrite=False, from_top=True):
149 if self.typus is not SkeletonTypus.Root and from_top is True:
150 if self.parent is not None:
151 self.parent.start_build(overwrite, from_top)
152 else:
153 raise AttributeError("This method with the argument 'from_top'=True can not be used on items that are not linked utimately to an ROOT Item")
154 else:
155 self._build(overwrite)
157 def serialize(self, save_file_path, strategy='json', from_top=True):
158 if self.typus is not SkeletonTypus.Root and from_top is True:
159 if self.parent is not None:
160 self.parent.serialize(save_file_path, strategy, from_top)
161 else:
162 raise AttributeError("This method with the argument 'from_top'=True can not be used on items that are not linked utimately to an ROOT Item")
163 elif self.typus is not SkeletonTypus.Root and from_top is False:
164 self.parent = None
165 _func, _extension = self.serialize_strategies[strategy]
166 _file = save_file_path + _extension if '.' not in os.path.basename(save_file_path) else save_file_path
167 _func(self, _file)
168 elif self.typus is SkeletonTypus.Root:
169 _func, _extension = self.serialize_strategies[strategy]
170 _file = save_file_path + _extension if '.' not in os.path.basename(save_file_path) else save_file_path
171 _func(self, _file)
173 @classmethod
174 def from_json_file(cls, json_file_path):
175 data = loadjson(json_file_path)
176 return cls.from_dict(data)
178 @classmethod
179 def from_dict(cls, dict_data):
180 root = cls(name=cls.standard_root_name, typus=SkeletonTypus.Root)
181 for parent, value in dict_data.items():
182 for name, typus, content in value:
183 if parent == cls.standard_root_name:
184 root.add_child_item(SkeletonInstructionItem(name=name, typus=SkeletonTypus(typus), content=content))
185 else:
186 root.find_node(parent).add_child_item(SkeletonInstructionItem(name=name, typus=SkeletonTypus(typus), content=content))
187 return root
189 def __getattr__(self, name):
190 _out = None
191 for child in self.children:
192 if self._make_child_keyname(child).casefold() == name.casefold():
193 _out = child
194 if _out is None:
195 raise AttributeError(name)
196 return _out
198 def all_nodes(self):
199 if self.parent is not None:
200 self.parent.all_nodes()
201 else:
202 yield self.name, self
203 yield from self._all_nodes_walker()
205 def _all_nodes_walker(self):
206 if self.typus is not SkeletonTypus.File:
207 for child_item in self.children:
208 yield child_item.name, child_item
209 yield from child_item._all_nodes_walker()
211 def find_node(self, name):
212 for node_name, node_object in self.all_nodes():
213 if node_name.casefold() == name.casefold():
214 return node_object
217class DirSkeletonReader:
218 exclude = {'__pycache__', '.git'}
219 exclude = set(map(lambda x: x.casefold(), exclude))
220 prebuilt_basefolder = pathmaker(THIS_FILE_DIR, '../data/skeletons')
221 prebuilt_folders = loadjson(pathmaker(THIS_FILE_DIR, 'prebuilt_folders.json'))
222 serialized_prebuilts_folder = pathmaker(THIS_FILE_DIR, '../data/serialized_skeletons')
224 def __init__(self, start_folder_path):
225 self.start_folder_path = start_folder_path
226 self.start_folder_name = os.path.basename(self.start_folder_path)
227 self.skeleton_tree = None
228 self._make_skeleton_tree()
230 def _make_skeleton_tree(self):
231 self.skeleton_tree = SkeletonInstructionItem(self.start_folder_name, SkeletonTypus.Root)
232 for dirname, folderlist, filelist in os.walk(self.start_folder_path, followlinks=False):
233 if all(ex_item.casefold() not in dirname.casefold() for ex_item in self.exclude):
234 parent = os.path.basename(dirname)
235 for foldername in folderlist:
236 if foldername.casefold() not in self.exclude:
237 self.skeleton_tree.find_node(parent).add_child_item(SkeletonInstructionItem(foldername, SkeletonTypus.Folder))
238 for filename in filelist:
239 if filename.casefold() not in self.exclude:
240 self.skeleton_tree.find_node(parent).add_child_item(SkeletonInstructionItem(filename, SkeletonTypus.File, content=read_file(pathmaker(dirname, filename))))
242 def rename_node(self, node_name, new_name):
243 node = self.find_node(node_name)
244 node.name = new_name
246 def __getattr__(self, name):
247 return getattr(self.skeleton_tree, name)
250def quick_check_skeleton():
251 item_dict = {
252 'first_folder': [('first_subfolder', SkeletonTypus.Folder, None),
253 ('second_subfolder', SkeletonTypus.Folder, None),
254 ('first_folder_image_file.jpg', SkeletonTypus.File, readbin(r"D:\Dropbox\hobby\Modding\Programs\Github\My_Repos\GidAppData\tests\skeleton_tree_test\afa_logoover_ca.jpg")),
255 ('first_folder_ini_file.ini', SkeletonTypus.File, readit(r"D:\Dropbox\hobby\Modding\Programs\Github\My_Repos\GidAppData\tests\skeleton_tree_test\example_cfg.ini"))],
256 'second_folder': [('something_file.txt', SkeletonTypus.File, 'this')]}
257 root = SkeletonInstructionItem(name='root', typus=SkeletonTypus.Root)
258 root.add_child_item(SkeletonInstructionItem(name='first_folder', typus=SkeletonTypus.Folder))
259 root.add_child_item(SkeletonInstructionItem(name='second_folder', typus=SkeletonTypus.Folder))
260 root.add_child_item(SkeletonInstructionItem(name='text_file.txt', typus=SkeletonTypus.File, content='this is a test text'))
262 for key, value in item_dict.items():
263 for _name, _typus, _content in value:
264 root.children[key].add_child_item(SkeletonInstructionItem(name=_name, typus=_typus, content=_content))
265 root.first_folder.first_subfolder.add_child_item(SkeletonInstructionItem(name='nested_test.json', typus=SkeletonTypus.File, content='{}'))
266 print(root.find_node('first_folder_ini_file.ini').name)
269def get_all_prebuilts():
270 skeleton_selections = {}
271 for prebuilt_typus in DirSkeletonReader.prebuilt_folders:
272 src_folder = pathmaker(DirSkeletonReader.prebuilt_basefolder, prebuilt_typus)
273 if src_folder not in skeleton_selections:
274 skeleton_selections[prebuilt_typus] = {}
275 for skeleton in os.scandir(src_folder):
276 if os.path.isdir(skeleton.path):
277 skeleton_selections[prebuilt_typus][skeleton.name] = pathmaker(skeleton.path, 'data_pack')
278 return skeleton_selections
281def serialize_all_prebuilts():
282 for file in os.scandir(DirSkeletonReader.serialized_prebuilts_folder):
283 if os.path.isfile(file):
284 os.remove(file.path)
285 for typus_folder, value in get_all_prebuilts().items():
286 typus = typus_folder.replace('prebuilt_', '')
287 for name, path in value.items():
288 name = f"[{typus}]_{name}"
289 skeleton_tree = DirSkeletonReader(path)
290 skeleton_tree.serialize(pathmaker(DirSkeletonReader.serialized_prebuilts_folder, name))
293# region[Main_Exec]
296if __name__ == '__main__':
297 pass
298# endregion[Main_Exec]