Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import re 

2import sys 

3import types 

4import importlib 

5 

6from .astutil import parse 

7from .astutil import store 

8from .astutil import load 

9from .astutil import ItemLookupOnAttributeErrorVisitor 

10from .codegen import TemplateCodeGenerator 

11from .codegen import template 

12from .codegen import reverse_builtin_map 

13from .astutil import Builtin 

14from .astutil import Symbol 

15from .exc import ExpressionError 

16from .utils import ast 

17from .utils import resolve_dotted 

18from .utils import ImportableMarker 

19from .utils import Markup 

20from .tokenize import Token 

21from .parser import substitute 

22from .compiler import Interpolator 

23 

24DEFAULT_MARKER = ImportableMarker(__name__, "DEFAULT") 

25 

26try: 

27 from .py26 import lookup_attr 

28except SyntaxError: 

29 from .py25 import lookup_attr 

30 

31 

32split_parts = re.compile(r'(?<!\\)\|') 

33match_prefix = re.compile(r'^\s*([a-z][a-z0-9\-_]*):').match 

34re_continuation = re.compile(r'\\\s*$', re.MULTILINE) 

35 

36try: 

37 from __builtin__ import basestring 

38except ImportError: 

39 basestring = str 

40 

41exc_clear = getattr(sys, "exc_clear", None) 

42 

43 

44def resolve_global(value): 

45 name = reverse_builtin_map.get(value) 

46 if name is not None: 

47 return Builtin(name) 

48 

49 return Symbol(value) 

50 

51 

52def test(expression, engine=None, **env): 

53 if engine is None: 

54 engine = SimpleEngine() 

55 

56 body = expression(store("result"), engine) 

57 module = ast.Module(body) 

58 module = ast.fix_missing_locations(module) 

59 env['rcontext'] = {} 

60 if exc_clear is not None: 

61 env['__exc_clear'] = exc_clear 

62 source = TemplateCodeGenerator(module).code 

63 code = compile(source, '<string>', 'exec') 

64 exec(code, env) 

65 result = env["result"] 

66 

67 if isinstance(result, basestring): 

68 result = str(result) 

69 

70 return result 

71 

72 

73def transform_attribute(node): 

74 return template( 

75 "lookup(object, name)", 

76 lookup=Symbol(lookup_attr), 

77 object=node.value, 

78 name=ast.Str(s=node.attr), 

79 mode="eval" 

80 ) 

81 

82 

83class TalesExpr(object): 

84 """Base class. 

85 

86 This class helps implementations for the Template Attribute 

87 Language Expression Syntax (TALES). 

88 

89 The syntax evaluates one or more expressions, separated by '|' 

90 (pipe). The first expression that succeeds, is returned. 

91 

92 Expression: 

93 

94 expression := (type ':')? line ('|' expression)? 

95 line := .* 

96 

97 Expression lines may not contain the pipe character unless 

98 escaped. It has a special meaning: 

99 

100 If the expression to the left of the pipe fails (raises one of the 

101 exceptions listed in ``catch_exceptions``), evaluation proceeds to 

102 the expression(s) on the right. 

103 

104 Subclasses must implement ``translate`` which assigns a value for 

105 a given expression. 

106 

107 >>> class PythonPipeExpr(TalesExpr): 

108 ... def translate(self, expression, target): 

109 ... compiler = PythonExpr(expression) 

110 ... return compiler(target, None) 

111 

112 >>> test(PythonPipeExpr('foo | bar | 42')) 

113 42 

114 

115 >>> test(PythonPipeExpr('foo|42')) 

116 42 

117 """ 

118 

119 exceptions = NameError, \ 

120 ValueError, \ 

121 AttributeError, \ 

122 LookupError, \ 

123 TypeError 

124 

125 ignore_prefix = True 

126 

127 def __init__(self, expression): 

128 self.expression = expression 

129 

130 def __call__(self, target, engine): 

131 remaining = self.expression 

132 assignments = [] 

133 

134 while remaining: 

135 if self.ignore_prefix and match_prefix(remaining) is not None: 

136 compiler = engine.parse(remaining) 

137 assignment = compiler.assign_value(target) 

138 remaining = "" 

139 else: 

140 for m in split_parts.finditer(remaining): 

141 expression = remaining[:m.start()] 

142 remaining = remaining[m.end():] 

143 break 

144 else: 

145 expression = remaining 

146 remaining = "" 

147 

148 expression = expression.replace('\\|', '|') 

149 assignment = self.translate_proxy(engine, expression, target) 

150 assignments.append(assignment) 

151 

152 if not assignments: 

153 if not remaining: 

154 raise ExpressionError("No input:", remaining) 

155 

156 assignments.append( 

157 self.translate_proxy(engine, remaining, target) 

158 ) 

159 

160 for i, assignment in enumerate(reversed(assignments)): 

161 if i == 0: 

162 body = assignment 

163 else: 

164 body = [ast.TryExcept( 

165 body=assignment, 

166 handlers=[ast.ExceptHandler( 

167 type=ast.Tuple( 

168 elts=map(resolve_global, self.exceptions), 

169 ctx=ast.Load()), 

170 name=None, 

171 body=body if exc_clear is None else body + [ 

172 ast.Expr( 

173 ast.Call( 

174 func=load("__exc_clear"), 

175 args=[], 

176 keywords=[], 

177 starargs=None, 

178 kwargs=None, 

179 ) 

180 ) 

181 ], 

182 )], 

183 )] 

184 

185 return body 

186 

187 def translate_proxy(self, engine, *args): 

188 """Default implementation delegates to ``translate`` method.""" 

189 

190 return self.translate(*args) 

191 

192 def translate(self, expression, target): 

193 """Return statements that assign a value to ``target``.""" 

194 

195 raise NotImplementedError( 

196 "Must be implemented by a subclass.") 

197 

198 

199class PathExpr(TalesExpr): 

200 """Path expression compiler. 

201 

202 Syntax:: 

203 

204 PathExpr ::= Path [ '|' Path ]* 

205 Path ::= variable [ '/' URL_Segment ]* 

206 variable ::= Name 

207 

208 For example:: 

209 

210 request/cookies/oatmeal 

211 nothing 

212 here/some-file 2001_02.html.tar.gz/foo 

213 root/to/branch | default 

214 

215 When a path expression is evaluated, it attempts to traverse 

216 each path, from left to right, until it succeeds or runs out of 

217 paths. To traverse a path, it first fetches the object stored in 

218 the variable. For each path segment, it traverses from the current 

219 object to the subobject named by the path segment. 

220 

221 Once a path has been successfully traversed, the resulting object 

222 is the value of the expression. If it is a callable object, such 

223 as a method or class, it is called. 

224 

225 The semantics of traversal (and what it means to be callable) are 

226 implementation-dependent (see the ``translate`` method). 

227 """ 

228 

229 def translate(self, expression, target): 

230 raise NotImplementedError( 

231 "Path expressions are not yet implemented. " 

232 "It's unclear whether a general implementation " 

233 "can be devised.") 

234 

235 

236class PythonExpr(TalesExpr): 

237 r"""Python expression compiler. 

238 

239 >>> test(PythonExpr('2 + 2')) 

240 4 

241 

242 The Python expression is a TALES expression. That means we can use 

243 the pipe operator: 

244 

245 >>> test(PythonExpr('foo | 2 + 2 | 5')) 

246 4 

247 

248 To include a pipe character, use a backslash escape sequence: 

249 

250 >>> test(PythonExpr(r'"\|"')) 

251 '|' 

252 """ 

253 

254 transform = ItemLookupOnAttributeErrorVisitor(transform_attribute) 

255 

256 def parse(self, string): 

257 return parse(string, 'eval').body 

258 

259 def translate(self, expression, target): 

260 # Strip spaces 

261 string = expression.strip() 

262 

263 # Conver line continuations to newlines 

264 string = substitute(re_continuation, '\n', string) 

265 

266 # Convert newlines to spaces 

267 string = string.replace('\n', ' ') 

268 

269 try: 

270 value = self.parse(string) 

271 except SyntaxError: 

272 exc = sys.exc_info()[1] 

273 raise ExpressionError(exc.msg, string) 

274 

275 # Transform attribute lookups to allow fallback to item lookup 

276 self.transform.visit(value) 

277 

278 return [ast.Assign(targets=[target], value=value)] 

279 

280 

281class ImportExpr(object): 

282 re_dotted = re.compile(r'^[A-Za-z.]+$') 

283 

284 def __init__(self, expression): 

285 self.expression = expression 

286 

287 def __call__(self, target, engine): 

288 string = self.expression.strip().replace('\n', ' ') 

289 value = template( 

290 "RESOLVE(NAME)", 

291 RESOLVE=Symbol(resolve_dotted), 

292 NAME=ast.Str(s=string), 

293 mode="eval", 

294 ) 

295 return [ast.Assign(targets=[target], value=value)] 

296 

297 

298class NotExpr(object): 

299 """Negates the expression. 

300 

301 >>> engine = SimpleEngine(PythonExpr) 

302 

303 >>> test(NotExpr('False'), engine) 

304 True 

305 >>> test(NotExpr('True'), engine) 

306 False 

307 """ 

308 

309 def __init__(self, expression): 

310 self.expression = expression 

311 

312 def __call__(self, target, engine): 

313 compiler = engine.parse(self.expression) 

314 body = compiler.assign_value(target) 

315 return body + template("target = not target", target=target) 

316 

317 

318class StructureExpr(object): 

319 """Wraps the expression result as 'structure'. 

320 

321 >>> engine = SimpleEngine(PythonExpr) 

322 

323 >>> test(StructureExpr('\"<tt>foo</tt>\"'), engine) 

324 '<tt>foo</tt>' 

325 """ 

326 

327 wrapper_class = Symbol(Markup) 

328 

329 def __init__(self, expression): 

330 self.expression = expression 

331 

332 def __call__(self, target, engine): 

333 compiler = engine.parse(self.expression) 

334 body = compiler.assign_value(target) 

335 return body + template( 

336 "target = wrapper(target)", 

337 target=target, 

338 wrapper=self.wrapper_class 

339 ) 

340 

341 

342class IdentityExpr(object): 

343 """Identity expression. 

344 

345 Exists to demonstrate the interface. 

346 

347 >>> test(IdentityExpr('42')) 

348 42 

349 """ 

350 

351 def __init__(self, expression): 

352 self.expression = expression 

353 

354 def __call__(self, target, engine): 

355 compiler = engine.parse(self.expression) 

356 return compiler.assign_value(target) 

357 

358 

359class StringExpr(object): 

360 """Similar to the built-in ``string.Template``, but uses an 

361 

362 expression engine to support pluggable string substitution 

363 expressions. 

364 

365 Expr string: 

366 

367 string := (text | substitution) (string)? 

368 substitution := ('$' variable | '${' expression '}') 

369 text := .* 

370 

371 In other words, an expression string can contain multiple 

372 substitutions. The text- and substitution parts will be 

373 concatenated back into a string. 

374 

375 >>> test(StringExpr('Hello ${name}!'), name='world') 

376 'Hello world!' 

377 

378 In the default configuration, braces may be omitted if the 

379 expression is an identifier. 

380 

381 >>> test(StringExpr('Hello $name!'), name='world') 

382 'Hello world!' 

383 

384 The ``braces_required`` flag changes this setting: 

385 

386 >>> test(StringExpr('Hello $name!', True)) 

387 'Hello $name!' 

388 

389 To avoid interpolation, use two dollar symbols. Note that only a 

390 single symbol will appear in the output. 

391 

392 >>> test(StringExpr('$${name}')) 

393 '${name}' 

394 

395 In previous versions, it was possible to escape using a regular 

396 backslash coding, but this is no longer supported. 

397 

398 >>> test(StringExpr(r'\\${name}'), name='Hello world!') 

399 '\\\\Hello world!' 

400 

401 Multiple interpolations in one: 

402 

403 >>> test(StringExpr("Hello ${'a'}${'b'}${'c'}!")) 

404 'Hello abc!' 

405 

406 Here's a more involved example taken from a javascript source: 

407 

408 >>> result = test(StringExpr(\"\"\" 

409 ... function($$, oid) { 

410 ... $('#' + oid).autocomplete({source: ${'source'}}); 

411 ... } 

412 ... \"\"\")) 

413 

414 >>> 'source: source' in result 

415 True 

416 

417 As noted previously, the double-dollar escape also affects 

418 non-interpolation expressions. 

419 

420 >>> 'function($, oid)' in result 

421 True 

422 

423 >>> test(StringExpr('test ${1}${2}')) 

424 'test 12' 

425 

426 >>> test(StringExpr('test $${1}${2}')) 

427 'test ${1}2' 

428 

429 >>> test(StringExpr('test $$')) 

430 'test $' 

431 

432 >>> test(StringExpr('$$.ajax(...)')) 

433 '$.ajax(...)' 

434 

435 >>> test(StringExpr('test $$ ${1}')) 

436 'test $ 1' 

437 

438 In the above examples, the expression is evaluated using the 

439 dummy engine which just returns the input as a string. 

440 

441 As an example, we'll implement an expression engine which 

442 instead counts the number of characters in the expresion and 

443 returns an integer result. 

444 

445 >>> class engine: 

446 ... @staticmethod 

447 ... def parse(expression, char_escape=None): 

448 ... class compiler: 

449 ... @staticmethod 

450 ... def assign_text(target): 

451 ... return [ 

452 ... ast.Assign( 

453 ... targets=[target], 

454 ... value=ast.Num(n=len(expression)) 

455 ... )] 

456 ... 

457 ... return compiler 

458 

459 This will demonstrate how the string expression coerces the 

460 input to a string. 

461 

462 >>> expr = StringExpr( 

463 ... 'There are ${hello world} characters in \"hello world\"') 

464 

465 We evaluate the expression using the new engine: 

466 

467 >>> test(expr, engine) 

468 'There are 11 characters in \"hello world\"' 

469 """ 

470 

471 def __init__(self, expression, braces_required=False): 

472 # The code relies on the expression being a token string 

473 if not isinstance(expression, Token): 

474 expression = Token(expression, 0) 

475 

476 self.translator = Interpolator(expression, braces_required) 

477 

478 def __call__(self, name, engine): 

479 return self.translator(name, engine) 

480 

481 

482class ProxyExpr(TalesExpr): 

483 braces_required = False 

484 

485 def __init__(self, name, expression, ignore_prefix=True): 

486 super(ProxyExpr, self).__init__(expression) 

487 self.ignore_prefix = ignore_prefix 

488 self.name = name 

489 

490 def translate_proxy(self, engine, expression, target): 

491 translator = Interpolator(expression, self.braces_required) 

492 assignment = translator(target, engine) 

493 

494 return assignment + [ 

495 ast.Assign(targets=[target], value=ast.Call( 

496 func=load(self.name), 

497 args=[target], 

498 keywords=[], 

499 starargs=None, 

500 kwargs=None 

501 )) 

502 ] 

503 

504 

505class ExistsExpr(object): 

506 """Boolean wrapper. 

507 

508 Return 0 if the expression results in an exception, otherwise 1. 

509 

510 As a means to generate exceptions, we set up an expression engine 

511 which evaluates the provided expression using Python: 

512 

513 >>> engine = SimpleEngine(PythonExpr) 

514 

515 >>> test(ExistsExpr('int(0)'), engine) 

516 1 

517 >>> test(ExistsExpr('int(None)'), engine) 

518 0 

519 

520 """ 

521 

522 exceptions = AttributeError, LookupError, TypeError, NameError, KeyError 

523 

524 def __init__(self, expression): 

525 self.expression = expression 

526 

527 def __call__(self, target, engine): 

528 ignore = store("_ignore") 

529 compiler = engine.parse(self.expression, False) 

530 body = compiler.assign_value(ignore) 

531 

532 classes = map(resolve_global, self.exceptions) 

533 

534 return [ 

535 ast.TryExcept( 

536 body=body, 

537 handlers=[ast.ExceptHandler( 

538 type=ast.Tuple(elts=classes, ctx=ast.Load()), 

539 name=None, 

540 body=template("target = 0", target=target), 

541 )], 

542 orelse=template("target = 1", target=target) 

543 ) 

544 ] 

545 

546 

547class ExpressionParser(object): 

548 def __init__(self, factories, default): 

549 self.factories = factories 

550 self.default = default 

551 

552 def __call__(self, expression): 

553 m = match_prefix(expression) 

554 if m is not None: 

555 prefix = m.group(1) 

556 expression = expression[m.end():] 

557 else: 

558 prefix = self.default 

559 

560 try: 

561 factory = self.factories[prefix] 

562 except KeyError: 

563 exc = sys.exc_info()[1] 

564 raise LookupError( 

565 "Unknown expression type: %s." % str(exc) 

566 ) 

567 

568 return factory(expression) 

569 

570 

571class SimpleEngine(object): 

572 expression = PythonExpr 

573 

574 def __init__(self, expression=None): 

575 if expression is not None: 

576 self.expression = expression 

577 

578 def parse(self, string, handle_errors=False, char_escape=None): 

579 compiler = self.expression(string) 

580 return SimpleCompiler(compiler, self) 

581 

582 

583class SimpleCompiler(object): 

584 def __init__(self, compiler, engine): 

585 self.compiler = compiler 

586 self.engine = engine 

587 

588 def assign_text(self, target): 

589 """Assign expression string as a text value.""" 

590 

591 return self._assign_value_and_coerce(target, "str") 

592 

593 def assign_value(self, target): 

594 """Assign expression string as object value.""" 

595 

596 return self.compiler(target, self.engine) 

597 

598 def _assign_value_and_coerce(self, target, builtin): 

599 return self.assign_value(target) + template( 

600 "target = builtin(target)", 

601 target=target, 

602 builtin=builtin 

603 )