Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# region [Imports] 

2 

3# * Standard Library Imports --> 

4import gc 

5import os 

6import re 

7import sys 

8import json 

9import lzma 

10import time 

11import queue 

12import logging 

13import platform 

14import subprocess 

15from enum import Enum, Flag, auto 

16from time import sleep 

17from pprint import pprint, pformat 

18from typing import Union 

19from datetime import tzinfo, datetime, timezone, timedelta 

20from functools import wraps, lru_cache, singledispatch, total_ordering, partial 

21from contextlib import contextmanager 

22from collections import Counter, ChainMap, deque, namedtuple, defaultdict 

23from multiprocessing import Pool 

24from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor 

25 

26# * Third Party Imports --> 

27# import requests 

28# import pyperclip 

29# import matplotlib.pyplot as plt 

30# from bs4 import BeautifulSoup 

31# from dotenv import load_dotenv 

32# from github import Github, GithubException 

33# from jinja2 import BaseLoader, Environment 

34# from natsort import natsorted 

35# from fuzzywuzzy import fuzz, process 

36 

37# * Gid Imports --> 

38import gidlogger as glog 

39 

40from gidappdata.utility import readit, readbin, writebin, writeit, writejson, loadjson, linereadit, pathmaker, create_folder, create_file, clearit, pickleit, get_pickled, read_file 

41import base64 

42 

43# endregion[Imports] 

44 

45 

46# region [AppUserData] 

47 

48# endregion [AppUserData] 

49 

50# region [Logging] 

51 

52log = logging.getLogger('gidappdata') 

53log.info(glog.imported(__name__)) 

54 

55# endregion[Logging] 

56 

57THIS_FILE_DIR = os.path.abspath(os.path.dirname(__file__)) 

58 

59 

60def json_parts(item): 

61 if item.typus is not SkeletonTypus.Root: 

62 yield (item.parent.name, item.name, item.typus.value, item.content) 

63 

64 if item.typus is not SkeletonTypus.File: 

65 for child in item.children: 

66 yield from json_parts(child) 

67 

68 

69def json_it(item, _file): 

70 _out = {} 

71 for _parent, _name, _typus, _content in json_parts(item): 

72 if _parent not in _out: 

73 _out[_parent] = [] 

74 if isinstance(_content, bytes): 

75 _content = base64.b64encode(_content) 

76 _out[_parent].append((_name, _typus, _content)) 

77 writejson(_out, _file, indent=2, sort_keys=False) 

78 

79 

80class SkeletonTypus(Enum): 

81 Folder = 'folder' 

82 File = 'file' 

83 Root = 'root' 

84 

85 

86class SkeletonInstructionBaseItem: 

87 pass 

88 

89 

90class SkeletonInstructionItem(SkeletonInstructionBaseItem): 

91 serialize_strategies = {'pickle': (pickleit, '.pkl'), 'json': (json_it, '.json')} 

92 standard_root_name = 'data_pack' 

93 

94 def __init__(self, name: str, typus: SkeletonTypus, parent: SkeletonInstructionBaseItem = None, content=None): 

95 self.name = name 

96 self.typus = typus 

97 self.parent = parent 

98 self.children = [] if self.typus in [SkeletonTypus.Folder, SkeletonTypus.Root] else None 

99 self.content = content 

100 

101 @staticmethod 

102 def _make_child_keyname(child): 

103 return child.name.replace('.', '__') 

104 

105 def add_child_item(self, new_child: SkeletonInstructionBaseItem): 

106 if self.typus is SkeletonTypus.File: 

107 raise AttributeError("Files can not have children Items") 

108 if any(new_child.name.casefold() == existing_child.name.casefold() for existing_child in self.children): 

109 raise FileExistsError(f"The {new_child.typus.value} {new_child.name} already exist inside this Folder") 

110 self.children.append(new_child) 

111 new_child.parent = self 

112 

113 @property 

114 def path(self): 

115 if self.parent is None or self.typus is SkeletonTypus.Root: 

116 return pathmaker(self.name) 

117 else: 

118 return pathmaker(self.parent.path, self.name) 

119 

120 def set_root_path(self, root_path): 

121 if self.typus is SkeletonTypus.Root: 

122 self.name = root_path 

123 elif self.parent is None: 

124 raise AttributeError("This method can not be used on items that are not linked utimately to an ROOT Item") 

125 else: 

126 self.parent.set_root_path(root_path) 

127 

128 def get_paths(self): 

129 yield self.path 

130 if self.typus is not SkeletonTypus.File: 

131 for child in self.children: 

132 yield from child.get_paths() 

133 

134 def _build(self, overwrite=False): 

135 if self.typus is SkeletonTypus.Root: 

136 create_folder(self.path, False) 

137 elif self.typus is SkeletonTypus.Folder: 

138 create_folder(self.path, overwrite=overwrite) 

139 

140 elif self.typus is SkeletonTypus.File: 

141 content = '' if self.content is None else self.content 

142 create_file(self.path, content, overwrite=overwrite) 

143 

144 if self.children is not None: 

145 for child in self.children: 

146 child._build(overwrite=overwrite) 

147 

148 def start_build(self, overwrite=False, from_top=True): 

149 if self.typus is not SkeletonTypus.Root and from_top is True: 

150 if self.parent is not None: 

151 self.parent.start_build(overwrite, from_top) 

152 else: 

153 raise AttributeError("This method with the argument 'from_top'=True can not be used on items that are not linked utimately to an ROOT Item") 

154 else: 

155 self._build(overwrite) 

156 

157 def serialize(self, save_file_path, strategy='json', from_top=True): 

158 if self.typus is not SkeletonTypus.Root and from_top is True: 

159 if self.parent is not None: 

160 self.parent.serialize(save_file_path, strategy, from_top) 

161 else: 

162 raise AttributeError("This method with the argument 'from_top'=True can not be used on items that are not linked utimately to an ROOT Item") 

163 elif self.typus is not SkeletonTypus.Root and from_top is False: 

164 self.parent = None 

165 _func, _extension = self.serialize_strategies[strategy] 

166 _file = save_file_path + _extension if '.' not in os.path.basename(save_file_path) else save_file_path 

167 _func(self, _file) 

168 elif self.typus is SkeletonTypus.Root: 

169 _func, _extension = self.serialize_strategies[strategy] 

170 _file = save_file_path + _extension if '.' not in os.path.basename(save_file_path) else save_file_path 

171 _func(self, _file) 

172 

173 @classmethod 

174 def from_json_file(cls, json_file_path): 

175 data = loadjson(json_file_path) 

176 return cls.from_dict(data) 

177 

178 @classmethod 

179 def from_dict(cls, dict_data): 

180 root = cls(name=cls.standard_root_name, typus=SkeletonTypus.Root) 

181 for parent, value in dict_data.items(): 

182 for name, typus, content in value: 

183 if parent == cls.standard_root_name: 

184 root.add_child_item(SkeletonInstructionItem(name=name, typus=SkeletonTypus(typus), content=content)) 

185 else: 

186 root.find_node(parent).add_child_item(SkeletonInstructionItem(name=name, typus=SkeletonTypus(typus), content=content)) 

187 return root 

188 

189 def __getattr__(self, name): 

190 _out = None 

191 for child in self.children: 

192 if self._make_child_keyname(child).casefold() == name.casefold(): 

193 _out = child 

194 if _out is None: 

195 raise AttributeError(name) 

196 return _out 

197 

198 def all_nodes(self): 

199 if self.parent is not None: 

200 self.parent.all_nodes() 

201 else: 

202 yield self.name, self 

203 yield from self._all_nodes_walker() 

204 

205 def _all_nodes_walker(self): 

206 if self.typus is not SkeletonTypus.File: 

207 for child_item in self.children: 

208 yield child_item.name, child_item 

209 yield from child_item._all_nodes_walker() 

210 

211 def find_node(self, name): 

212 for node_name, node_object in self.all_nodes(): 

213 if node_name.casefold() == name.casefold(): 

214 return node_object 

215 

216 

217class DirSkeletonReader: 

218 exclude = {'__pycache__', '.git'} 

219 exclude = set(map(lambda x: x.casefold(), exclude)) 

220 prebuilt_basefolder = pathmaker(THIS_FILE_DIR, '../data/skeletons') 

221 prebuilt_folders = loadjson(pathmaker(THIS_FILE_DIR, 'prebuilt_folders.json')) 

222 serialized_prebuilts_folder = pathmaker(THIS_FILE_DIR, '../data/serialized_skeletons') 

223 

224 def __init__(self, start_folder_path): 

225 self.start_folder_path = start_folder_path 

226 self.start_folder_name = os.path.basename(self.start_folder_path) 

227 self.skeleton_tree = None 

228 self._make_skeleton_tree() 

229 

230 def _make_skeleton_tree(self): 

231 self.skeleton_tree = SkeletonInstructionItem(self.start_folder_name, SkeletonTypus.Root) 

232 for dirname, folderlist, filelist in os.walk(self.start_folder_path, followlinks=False): 

233 if all(ex_item.casefold() not in dirname.casefold() for ex_item in self.exclude): 

234 parent = os.path.basename(dirname) 

235 for foldername in folderlist: 

236 if foldername.casefold() not in self.exclude: 

237 self.skeleton_tree.find_node(parent).add_child_item(SkeletonInstructionItem(foldername, SkeletonTypus.Folder)) 

238 for filename in filelist: 

239 if filename.casefold() not in self.exclude: 

240 self.skeleton_tree.find_node(parent).add_child_item(SkeletonInstructionItem(filename, SkeletonTypus.File, content=read_file(pathmaker(dirname, filename)))) 

241 

242 def rename_node(self, node_name, new_name): 

243 node = self.find_node(node_name) 

244 node.name = new_name 

245 

246 def __getattr__(self, name): 

247 return getattr(self.skeleton_tree, name) 

248 

249 

250def quick_check_skeleton(): 

251 item_dict = { 

252 'first_folder': [('first_subfolder', SkeletonTypus.Folder, None), 

253 ('second_subfolder', SkeletonTypus.Folder, None), 

254 ('first_folder_image_file.jpg', SkeletonTypus.File, readbin(r"D:\Dropbox\hobby\Modding\Programs\Github\My_Repos\GidAppData\tests\skeleton_tree_test\afa_logoover_ca.jpg")), 

255 ('first_folder_ini_file.ini', SkeletonTypus.File, readit(r"D:\Dropbox\hobby\Modding\Programs\Github\My_Repos\GidAppData\tests\skeleton_tree_test\example_cfg.ini"))], 

256 'second_folder': [('something_file.txt', SkeletonTypus.File, 'this')]} 

257 root = SkeletonInstructionItem(name='root', typus=SkeletonTypus.Root) 

258 root.add_child_item(SkeletonInstructionItem(name='first_folder', typus=SkeletonTypus.Folder)) 

259 root.add_child_item(SkeletonInstructionItem(name='second_folder', typus=SkeletonTypus.Folder)) 

260 root.add_child_item(SkeletonInstructionItem(name='text_file.txt', typus=SkeletonTypus.File, content='this is a test text')) 

261 

262 for key, value in item_dict.items(): 

263 for _name, _typus, _content in value: 

264 root.children[key].add_child_item(SkeletonInstructionItem(name=_name, typus=_typus, content=_content)) 

265 root.first_folder.first_subfolder.add_child_item(SkeletonInstructionItem(name='nested_test.json', typus=SkeletonTypus.File, content='{}')) 

266 print(root.find_node('first_folder_ini_file.ini').name) 

267 

268 

269def get_all_prebuilts(): 

270 skeleton_selections = {} 

271 for prebuilt_typus in DirSkeletonReader.prebuilt_folders: 

272 src_folder = pathmaker(DirSkeletonReader.prebuilt_basefolder, prebuilt_typus) 

273 if src_folder not in skeleton_selections: 

274 skeleton_selections[prebuilt_typus] = {} 

275 for skeleton in os.scandir(src_folder): 

276 if os.path.isdir(skeleton.path): 

277 skeleton_selections[prebuilt_typus][skeleton.name] = pathmaker(skeleton.path, 'data_pack') 

278 return skeleton_selections 

279 

280 

281def serialize_all_prebuilts(): 

282 for file in os.scandir(DirSkeletonReader.serialized_prebuilts_folder): 

283 if os.path.isfile(file): 

284 os.remove(file.path) 

285 for typus_folder, value in get_all_prebuilts().items(): 

286 typus = typus_folder.replace('prebuilt_', '') 

287 for name, path in value.items(): 

288 name = f"[{typus}]_{name}" 

289 skeleton_tree = DirSkeletonReader(path) 

290 skeleton_tree.serialize(pathmaker(DirSkeletonReader.serialized_prebuilts_folder, name)) 

291 

292 

293# region[Main_Exec] 

294 

295 

296if __name__ == '__main__': 

297 pass 

298# endregion[Main_Exec]