Coverage for /home/runner/.local/share/hatch/env/virtual/importnb/KA2AwMZG/test.interactive/lib/python3.9/site-packages/importnb/loader.py: 94%

229 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-24 22:30 +0000

1# coding: utf-8 

2"""# `loader` 

3 

4Combine the __import__ finder with the loader. 

5""" 

6 

7 

8import ast 

9from contextlib import contextmanager 

10import re 

11import sys 

12from telnetlib import DO 

13import textwrap 

14from dataclasses import asdict, dataclass, field 

15from functools import partial 

16from importlib import reload 

17from importlib._bootstrap import _init_module_attrs, _requires_builtin 

18from importlib._bootstrap_external import FileFinder, decode_source 

19from importlib.machinery import ModuleSpec, SourceFileLoader 

20from importlib.util import LazyLoader, find_spec 

21from pathlib import Path 

22from types import ModuleType 

23 

24from . import get_ipython 

25from .decoder import LineCacheNotebookDecoder, quote 

26from .docstrings import update_docstring 

27from .finder import FuzzyFinder, get_loader_details, get_loader_index 

28 

29_GTE38 = sys.version_info.major == 3 and sys.version_info.minor >= 8 

30 

31try: 

32 import IPython 

33 from IPython.core.inputsplitter import IPythonInputSplitter 

34 

35 dedent = IPythonInputSplitter( 

36 line_input_checker=False, 

37 physical_line_transforms=[ 

38 IPython.core.inputsplitter.leading_indent(), 

39 IPython.core.inputsplitter.ipy_prompt(), 

40 IPython.core.inputsplitter.cellmagic(end_on_blank_line=False), 

41 ], 

42 ).transform_cell 

43except ModuleNotFoundError: 

44 

45 def dedent(body): 

46 from textwrap import dedent, indent 

47 

48 if MAGIC.match(body): 

49 return indent(body, "# ") 

50 return dedent(body) 

51 

52 

53__all__ = "Notebook", "reload" 

54 

55 

56MAGIC = re.compile("^\s*%{2}", re.MULTILINE) 

57 

58 

59@dataclass 

60class Interface: 

61 name: str = None 

62 path: str = None 

63 lazy: bool = False 

64 extensions: tuple = field(default_factory=[".ipy", ".ipynb"].copy) 

65 include_fuzzy_finder: bool = True 

66 

67 include_markdown_docstring: bool = True 

68 only_defs: bool = False 

69 no_magic: bool = False 

70 _loader_hook_position: int = field(default=0, repr=False) 

71 

72 def __new__(cls, name=None, path=None, **kwargs): 

73 kwargs.update(name=name, path=path) 

74 self = super().__new__(cls) 

75 self.__init__(**kwargs) 

76 return self 

77 

78 

79class BaseLoader(Interface, SourceFileLoader): 

80 """The simplest implementation of a Notebook Source File Loader.""" 

81 

82 @property 

83 def loader(self): 

84 """Create a lazy loader source file loader.""" 

85 loader = type(self) 

86 if self.lazy and (sys.version_info.major, sys.version_info.minor) != (3, 4): 

87 loader = LazyLoader.factory(loader) 

88 # Strip the leading underscore from slots 

89 params = asdict(self) 

90 params.pop("name") 

91 params.pop("path") 

92 return partial(loader, **params) 

93 

94 @property 

95 def finder(self): 

96 """Permit fuzzy finding of files with special characters.""" 

97 return self.include_fuzzy_finder and FuzzyFinder or FileFinder 

98 

99 def translate(self, source): 

100 if self.path and self.path.endswith(".ipynb"): 

101 return LineCacheNotebookDecoder( 

102 code=self.code, raw=self.raw, markdown=self.markdown 

103 ).decode(source, self.path) 

104 return self.code(source) 

105 

106 def get_data(self, path): 

107 """Needs to return the string source for the module.""" 

108 return self.translate(self.decode()) 

109 

110 def create_module(self, spec): 

111 module = ModuleType(str(spec.name)) 

112 _init_module_attrs(spec, module) 

113 if self.name: 

114 module.__name__ = self.name 

115 if getattr(spec, "alias", None): 

116 # put a fuzzy spec on the modules to avoid re importing it. 

117 # there is a funky trick you do with the fuzzy finder where you 

118 # load multiple versions with different finders. 

119 

120 sys.modules[spec.alias] = module 

121 module.get_ipython = get_ipython 

122 return module 

123 

124 def decode(self): 

125 return decode_source(super().get_data(self.path)) 

126 

127 def code(self, str): 

128 return dedent(str) 

129 

130 def markdown(self, str): 

131 return quote(str) 

132 

133 def raw(self, str): 

134 return comment(str) 

135 

136 def visit(self, node): 

137 return node 

138 

139 @classmethod 

140 @_requires_builtin 

141 def is_package(cls, fullname): 

142 """Return False as built-in modules are never packages.""" 

143 if "." not in fullname: 

144 return True 

145 return super().is_package(fullname) 

146 

147 get_source = get_data 

148 

149 def __enter__(self): 

150 path_id, loader_id, details = get_loader_index(".py") 

151 for _, e in details: 

152 if all(map(e.__contains__, self.extensions)): 

153 self._loader_hook_position = None 

154 return self 

155 else: 

156 self._loader_hook_position = loader_id + 1 

157 details.insert(self._loader_hook_position, (self.loader, self.extensions)) 

158 sys.path_hooks[path_id] = self.finder.path_hook(*details) 

159 sys.path_importer_cache.clear() 

160 return self 

161 

162 def __exit__(self, *excepts): 

163 if self._loader_hook_position is not None: 

164 path_id, details = get_loader_details() 

165 details.pop(self._loader_hook_position) 

166 sys.path_hooks[path_id] = self.finder.path_hook(*details) 

167 sys.path_importer_cache.clear() 

168 

169 

170class FileModuleSpec(ModuleSpec): 

171 def __init__(self, *args, **kwargs): 

172 super().__init__(*args, **kwargs) 

173 self._set_fileattr = True 

174 

175 

176def comment(str): 

177 return textwrap.indent(str, "# ") 

178 

179 

180class DefsOnly(ast.NodeTransformer): 

181 INCLUDE = ast.Import, ast.ImportFrom, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef 

182 

183 def visit_Module(self, node): 

184 args = ([x for x in node.body if isinstance(x, self.INCLUDE)],) 

185 if _GTE38: 

186 args += (node.type_ignores,) 

187 return ast.Module(*args) 

188 

189 

190class Notebook(BaseLoader): 

191 """Notebook is a user friendly file finder and module loader for notebook source code. 

192 

193 > Remember, restart and run all or it didn't happen. 

194 

195 Notebook provides several useful options. 

196 

197 * Lazy module loading. A module is executed the first time it is used in a script. 

198 """ 

199 

200 def parse(self, nodes): 

201 return ast.parse(nodes, self.path) 

202 

203 def visit(self, nodes): 

204 if self.only_defs: 

205 nodes = DefsOnly().visit(nodes) 

206 return nodes 

207 

208 def code(self, str): 

209 if self.no_magic: 

210 if MAGIC.match(str): 

211 return comment(str) 

212 return super().code(str) 

213 

214 def source_to_code(self, nodes, path, *, _optimize=-1): 

215 """* Convert the current source to ast 

216 * Apply ast transformers. 

217 * Compile the code.""" 

218 if not isinstance(nodes, ast.Module): 

219 nodes = self.parse(nodes) 

220 if self.include_markdown_docstring: 

221 nodes = update_docstring(nodes) 

222 return super().source_to_code( 

223 ast.fix_missing_locations(self.visit(nodes)), path, _optimize=_optimize 

224 ) 

225 

226 @classmethod 

227 def load_file(cls, filename, main=True, **kwargs): 

228 """Import a notebook as a module from a filename. 

229 

230 dir: The directory to load the file from. 

231 main: Load the module in the __main__ context. 

232 

233 > assert Notebook.load('loader.ipynb') 

234 """ 

235 name = main and "__main__" or filename 

236 loader = cls(name, str(filename), **kwargs) 

237 spec = FileModuleSpec(name, loader, origin=loader.path) 

238 module = loader.create_module(spec) 

239 loader.exec_module(module) 

240 return module 

241 

242 @classmethod 

243 def load_module(cls, module, main=False, **kwargs): 

244 """Import a notebook as a module. 

245 

246 dir: The directory to load the file from. 

247 main: Load the module in the __main__ context. 

248 

249 > assert Notebook.load('loader.ipynb') 

250 """ 

251 from runpy import _run_module_as_main, run_module 

252 

253 with cls() as loader: 

254 if main: 

255 return _dict_module(_run_module_as_main(module)) 

256 else: 

257 spec = find_spec(module) 

258 

259 m = spec.loader.create_module(spec) 

260 spec.loader.exec_module(m) 

261 return m 

262 

263 @classmethod 

264 def load_argv(cls, argv=None, *, parser=None): 

265 import sys 

266 

267 if parser is None: 

268 parser = cls.get_argparser() 

269 

270 if argv is None: 

271 from sys import argv 

272 

273 argv = argv[1:] 

274 

275 if isinstance(argv, str): 

276 from shlex import split 

277 

278 argv = split(argv) 

279 

280 module = cls.load_ns(parser.parse_args(argv)) 

281 if module is None: 

282 return parser.print_help() 

283 

284 return module 

285 

286 @classmethod 

287 def load_ns(cls, ns): 

288 from sys import path 

289 

290 if ns.tasks: 

291 from doit.doit_cmd import DoitMain 

292 from doit.cmd_base import ModuleTaskLoader 

293 

294 if ns.code: 

295 with main_argv(sys.argv[0], ns.args): 

296 result = cls.load_code(ns.code) 

297 elif ns.module: 

298 path.insert(0, ns.dir) if ns.dir else ... if "" in path else path.insert(0, "") 

299 with main_argv(ns.module, ns.args): 

300 result = cls.load_module(ns.module, main=True) 

301 elif ns.file: 

302 where = Path(ns.dir, ns.file) if ns.dir else Path(ns.file) 

303 with main_argv(str(where), ns.args): 

304 result = cls.load_file(ns.file) 

305 else: 

306 return 

307 

308 if ns.tasks: 

309 DoitMain(ModuleTaskLoader(result)).run(ns.args) 

310 return result 

311 

312 @classmethod 

313 def load_code(cls, code, argv=None, mod_name=None, script_name=None, main=False): 

314 from runpy import _run_module_code 

315 

316 self = cls() 

317 name = main and "__main__" or mod_name or "<markdown code>" 

318 

319 return _dict_module( 

320 _run_module_code(self.translate(code), mod_name=name, script_name=script_name) 

321 ) 

322 

323 @staticmethod 

324 def get_argparser(parser=None): 

325 from argparse import REMAINDER, ArgumentParser 

326 

327 if parser is None: 

328 parser = ArgumentParser("importnb", description="run notebooks as python code") 

329 parser.add_argument("file", nargs="?", help="run a file") 

330 parser.add_argument("args", nargs=REMAINDER, help="arguments to pass to script") 

331 parser.add_argument("-m", "--module", help="run a module") 

332 parser.add_argument("-c", "--code", help="run raw code") 

333 parser.add_argument("-d", "--dir", help="path to run script in") 

334 parser.add_argument("-t", "--tasks", action="store_true", help="run doit tasks") 

335 return parser 

336 

337 

338def _dict_module(ns): 

339 m = ModuleType(ns.get("__name__"), ns.get("__doc__")) 

340 m.__dict__.update(ns) 

341 return m 

342 

343 

344@contextmanager 

345def main_argv(prog, args=None): 

346 if args is not None: 

347 if isinstance(args, str): 

348 from shlex import split 

349 

350 args = split(args) 

351 args = [prog] + args 

352 prior, sys.argv = sys.argv, args 

353 yield 

354 if args is not None: 

355 sys.argv = prior