homes/common/.local/lib/python2.7/site-packages/jedi/evaluate.py
Christophe Buffenoir 8385ed00af Add jedi
Fix error in powerline (ascii can't decode)
Get vim plugins directly from git
2013-05-03 10:58:55 +02:00

1614 lines
61 KiB
Python

"""
This is the core part of jedi. Most of the logic, how to evaluate certain
objects (imports, etc.) is here.
The functions should be described in their docstrings. However, there are some
classes, which are used to store the values. After those classes, there are the
search functions `get_names_for_scope` and `get_scopes_for_name`. At the end
there are the `follow_` functions, which evaluate a statement, or parts of a
statement.
TODO super()
TODO nonlocal statement, needed or can be ignored? (py3k)
"""
from _compatibility import next, property, hasattr, is_py3k, use_metaclass, \
unicode
import sys
import itertools
import copy
import parsing
import debug
import builtin
import imports
import helpers
import dynamic
import docstrings
memoize_caches = []
faked_scopes = []
class DecoratorNotFound(LookupError):
"""
Decorators are sometimes not found, if that happens, that error is raised.
"""
pass
class MultiLevelStopIteration(Exception):
"""
StopIteration's get catched pretty easy by for loops, let errors propagate.
"""
pass
class MultiLevelAttributeError(Exception):
"""
Important, because `__getattr__` and `hasattr` catch AttributeErrors
implicitly. This is really evil (mainly because of `__getattr__`).
`hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
Therefore this class has to be a `BaseException` and not an `Exception`.
But because I rewrote hasattr, we can now switch back to `Exception`.
:param base: return values of sys.exc_info().
"""
def __init__(self, base):
self.base = base
def __str__(self):
import traceback
tb = traceback.format_exception(*self.base)
return 'Original:\n\n' + ''.join(tb)
def clear_caches():
"""
Clears all caches of this and related modules. Jedi caches many things,
that should be completed after each completion finishes. The only things
that stays is the module cache (which is not deleted here).
"""
global memoize_caches, faked_scopes
for m in memoize_caches:
m.clear()
dynamic.search_param_cache.clear()
helpers.ExecutionRecursionDecorator.reset()
# memorize_caches must never be deleted, because the dicts will get lost in
# the wrappers.
faked_scopes = []
follow_statement.reset()
imports.imports_processed = 0
def memoize_default(default=None):
"""
This is a typical memoization decorator, BUT there is one difference:
To prevent recursion it sets defaults.
Preventing recursion is in this case the much bigger use than speed. I
don't think, that there is a big speed difference, but there are many cases
where recursion could happen (think about a = b; b = a).
"""
def func(function):
memo = {}
memoize_caches.append(memo)
def wrapper(*args, **kwargs):
key = (args, frozenset(kwargs.items()))
if key in memo:
return memo[key]
else:
memo[key] = default
rv = function(*args, **kwargs)
memo[key] = rv
return rv
return wrapper
return func
class CachedMetaClass(type):
"""
This is basically almost the same than the decorator above, it just caches
class initializations. I haven't found any other way, so I do it with meta
classes.
"""
@memoize_default()
def __call__(self, *args, **kwargs):
return super(CachedMetaClass, self).__call__(*args, **kwargs)
class Executable(parsing.Base):
""" An instance is also an executable - because __init__ is called """
def __init__(self, base, var_args=None):
self.base = base
# The param input array.
if var_args is None:
var_args = parsing.Array(None, None)
self.var_args = var_args
def get_parent_until(self, *args, **kwargs):
return self.base.get_parent_until(*args, **kwargs)
@property
def parent(self):
return self.base.parent
class Instance(use_metaclass(CachedMetaClass, Executable)):
""" This class is used to evaluate instances. """
def __init__(self, base, var_args=None):
super(Instance, self).__init__(base, var_args)
if str(base.name) in ['list', 'set'] \
and builtin.Builtin.scope == base.get_parent_until():
# compare the module path with the builtin name.
self.var_args = dynamic.check_array_instances(self)
else:
# need to execute the __init__ function, because the dynamic param
# searching needs it.
try:
self.execute_subscope_by_name('__init__', self.var_args)
except KeyError:
pass
# Generated instances are classes that are just generated by self
# (No var_args) used.
self.is_generated = False
@memoize_default()
def get_init_execution(self, func):
func = InstanceElement(self, func, True)
return Execution(func, self.var_args)
def get_func_self_name(self, func):
"""
Returns the name of the first param in a class method (which is
normally self
"""
try:
return func.params[0].used_vars[0].names[0]
except IndexError:
return None
def get_self_properties(self):
def add_self_dot_name(name):
n = copy.copy(name)
n.names = n.names[1:]
names.append(InstanceElement(self, n))
names = []
# This loop adds the names of the self object, copies them and removes
# the self.
for sub in self.base.subscopes:
if isinstance(sub, parsing.Class):
continue
# Get the self name, if there's one.
self_name = self.get_func_self_name(sub)
if self_name:
# Check the __init__ function.
if sub.name.get_code() == '__init__':
sub = self.get_init_execution(sub)
for n in sub.get_set_vars():
# Only names with the selfname are being added.
# It is also important, that they have a len() of 2,
# because otherwise, they are just something else
if n.names[0] == self_name and len(n.names) == 2:
add_self_dot_name(n)
for s in self.base.get_super_classes():
if s == self.base:
# I don't know how this could happen... But saw it once.
continue
names += Instance(s).get_self_properties()
return names
def get_subscope_by_name(self, name):
sub = self.base.get_subscope_by_name(name)
return InstanceElement(self, sub, True)
def execute_subscope_by_name(self, name, args=None):
if args is None:
args = helpers.generate_param_array([])
method = self.get_subscope_by_name(name)
if args.parent_stmt is None:
args.parent_stmt = method
return Execution(method, args).get_return_types()
def get_descriptor_return(self, obj):
""" Throws a KeyError if there's no method. """
# Arguments in __get__ descriptors are obj, class.
# `method` is the new parent of the array, don't know if that's good.
v = [obj, obj.base] if isinstance(obj, Instance) else [None, obj]
args = helpers.generate_param_array(v)
return self.execute_subscope_by_name('__get__', args)
@memoize_default([])
def get_defined_names(self):
"""
Get the instance vars of a class. This includes the vars of all
classes
"""
names = self.get_self_properties()
class_names = self.base.get_defined_names()
for var in class_names:
names.append(InstanceElement(self, var, True))
return names
def scope_generator(self):
"""
An Instance has two scopes: The scope with self names and the class
scope. Instance variables have priority over the class scope.
"""
yield self, self.get_self_properties()
names = []
class_names = self.base.get_defined_names()
for var in class_names:
names.append(InstanceElement(self, var, True))
yield self, names
def get_index_types(self, index=None):
args = helpers.generate_param_array([] if index is None else [index])
try:
return self.execute_subscope_by_name('__getitem__', args)
except KeyError:
debug.warning('No __getitem__, cannot access the array.')
return []
def __getattr__(self, name):
if name not in ['start_pos', 'end_pos', 'name', 'get_imports',
'docstr', 'asserts']:
raise AttributeError("Instance %s: Don't touch this (%s)!"
% (self, name))
return getattr(self.base, name)
def __repr__(self):
return "<e%s of %s (var_args: %s)>" % \
(type(self).__name__, self.base, len(self.var_args or []))
class InstanceElement(use_metaclass(CachedMetaClass)):
"""
InstanceElement is a wrapper for any object, that is used as an instance
variable (e.g. self.variable or class methods).
"""
def __init__(self, instance, var, is_class_var=False):
if isinstance(var, parsing.Function):
var = Function(var)
elif isinstance(var, parsing.Class):
var = Class(var)
self.instance = instance
self.var = var
self.is_class_var = is_class_var
@property
@memoize_default()
def parent(self):
par = self.var.parent
if isinstance(par, Class) and par == self.instance.base \
or isinstance(par, parsing.Class) \
and par == self.instance.base.base:
par = self.instance
elif not isinstance(par, parsing.Module):
par = InstanceElement(self.instance, par, self.is_class_var)
return par
def get_parent_until(self, *args, **kwargs):
return parsing.Simple.get_parent_until(self, *args, **kwargs)
def get_decorated_func(self):
""" Needed because the InstanceElement should not be stripped """
func = self.var.get_decorated_func()
if func == self.var:
return self
return func
def get_assignment_calls(self):
# Copy and modify the array.
origin = self.var.get_assignment_calls()
# Delete parent, because it isn't used anymore.
new = helpers.fast_parent_copy(origin)
par = InstanceElement(self.instance, origin.parent_stmt,
self.is_class_var)
new.parent_stmt = par
faked_scopes.append(par)
faked_scopes.append(new)
return new
def __getattr__(self, name):
return getattr(self.var, name)
def isinstance(self, *cls):
return isinstance(self.var, cls)
def __repr__(self):
return "<%s of %s>" % (type(self).__name__, self.var)
class Class(use_metaclass(CachedMetaClass, parsing.Base)):
"""
This class is not only important to extend `parsing.Class`, it is also a
important for descriptors (if the descriptor methods are evaluated or not).
"""
def __init__(self, base):
self.base = base
@memoize_default(default=[])
def get_super_classes(self):
supers = []
# TODO care for mro stuff (multiple super classes).
for s in self.base.supers:
# Super classes are statements.
for cls in follow_statement(s):
if not isinstance(cls, Class):
debug.warning('Received non class, as a super class')
continue # Just ignore other stuff (user input error).
supers.append(cls)
if not supers and self.base.parent != builtin.Builtin.scope:
# add `object` to classes
supers += get_scopes_for_name(builtin.Builtin.scope, 'object')
return supers
@memoize_default(default=[])
def get_defined_names(self):
def in_iterable(name, iterable):
""" checks if the name is in the variable 'iterable'. """
for i in iterable:
# Only the last name is important, because these names have a
# maximal length of 2, with the first one being `self`.
if i.names[-1] == name.names[-1]:
return True
return False
result = self.base.get_defined_names()
super_result = []
# TODO mro!
for cls in self.get_super_classes():
# Get the inherited names.
for i in cls.get_defined_names():
if not in_iterable(i, result):
super_result.append(i)
result += super_result
return result
def get_subscope_by_name(self, name):
for sub in reversed(self.subscopes):
if sub.name.get_code() == name:
return sub
raise KeyError("Couldn't find subscope.")
@property
def name(self):
return self.base.name
def __getattr__(self, name):
if name not in ['start_pos', 'end_pos', 'parent', 'subscopes',
'get_imports', 'get_parent_until', 'docstr', 'asserts']:
raise AttributeError("Don't touch this (%s)!" % name)
return getattr(self.base, name)
def __repr__(self):
return "<e%s of %s>" % (type(self).__name__, self.base)
class Function(use_metaclass(CachedMetaClass, parsing.Base)):
"""
Needed because of decorators. Decorators are evaluated here.
"""
def __init__(self, func, is_decorated=False):
""" This should not be called directly """
self.base_func = func
self.is_decorated = is_decorated
@property
@memoize_default()
def _decorated_func(self):
"""
Returns the function, that is to be executed in the end.
This is also the places where the decorators are processed.
"""
f = self.base_func
# Only enter it, if has not already been processed.
if not self.is_decorated:
for dec in reversed(self.base_func.decorators):
debug.dbg('decorator:', dec, f)
dec_results = follow_statement(dec)
if not len(dec_results):
debug.warning('decorator func not found: %s in stmt %s' %
(self.base_func, dec))
return None
if len(dec_results) > 1:
debug.warning('multiple decorators found', self.base_func,
dec_results)
decorator = dec_results.pop()
# Create param array.
old_func = Function(f, is_decorated=True)
params = helpers.generate_param_array([old_func], old_func)
faked_scopes.append(old_func)
wrappers = Execution(decorator, params).get_return_types()
if not len(wrappers):
debug.warning('no wrappers found', self.base_func)
return None
if len(wrappers) > 1:
debug.warning('multiple wrappers found', self.base_func,
wrappers)
# This is here, that the wrapper gets executed.
f = wrappers[0]
debug.dbg('decorator end', f)
if f != self.base_func and isinstance(f, parsing.Function):
f = Function(f)
return f
def get_decorated_func(self):
if self._decorated_func is None:
raise DecoratorNotFound()
if self._decorated_func == self.base_func:
return self
return self._decorated_func
def get_magic_method_names(self):
return builtin.Builtin.magic_function_scope.get_defined_names()
def get_magic_method_scope(self):
return builtin.Builtin.magic_function_scope
def __getattr__(self, name):
return getattr(self.base_func, name)
def __repr__(self):
dec = ''
if self._decorated_func != self.base_func:
dec = " is " + repr(self._decorated_func)
return "<e%s of %s%s>" % (type(self).__name__, self.base_func, dec)
class Execution(Executable):
"""
This class is used to evaluate functions and their returns.
This is the most complicated class, because it contains the logic to
transfer parameters. It is even more complicated, because there may be
multiple calls to functions and recursion has to be avoided. But this is
responsibility of the decorators.
"""
@memoize_default(default=[])
@helpers.ExecutionRecursionDecorator
def get_return_types(self, evaluate_generator=False):
""" Get the return types of a function. """
stmts = []
if self.base.parent == builtin.Builtin.scope \
and not isinstance(self.base, (Generator, Array)):
func_name = str(self.base.name)
# some implementations of builtins:
if func_name == 'getattr':
# follow the first param
try:
objects = follow_call_list([self.var_args[0]])
names = follow_call_list([self.var_args[1]])
except IndexError:
debug.warning('getattr() called with to few args.')
return []
for obj in objects:
if not isinstance(obj, (Instance, Class)):
debug.warning('getattr called without instance')
continue
for name in names:
key = name.var_args.get_only_subelement()
stmts += follow_path(iter([key]), obj, self.base)
return stmts
elif func_name == 'type':
# otherwise it would be a metaclass
if len(self.var_args) == 1:
objects = follow_call_list([self.var_args[0]])
return [o.base for o in objects if isinstance(o, Instance)]
if self.base.isinstance(Class):
# There maybe executions of executions.
stmts = [Instance(self.base, self.var_args)]
elif isinstance(self.base, Generator):
return self.base.iter_content()
else:
# Don't do this with exceptions, as usual, because some deeper
# exceptions could be catched - and I wouldn't know what happened.
try:
self.base.returns
except (AttributeError, DecoratorNotFound):
if hasattr(self.base, 'execute_subscope_by_name'):
try:
stmts = self.base.execute_subscope_by_name('__call__',
self.var_args)
except KeyError:
debug.warning("no __call__ func available", self.base)
else:
debug.warning("no execution possible", self.base)
else:
stmts = self._get_function_returns(evaluate_generator)
debug.dbg('exec result: %s in %s' % (stmts, self))
return imports.strip_imports(stmts)
def _get_function_returns(self, evaluate_generator):
""" A normal Function execution """
# Feed the listeners, with the params.
for listener in self.base.listeners:
listener.execute(self.get_params())
func = self.base.get_decorated_func()
if func.is_generator and not evaluate_generator:
return [Generator(func, self.var_args)]
else:
stmts = []
for r in self.returns:
stmts += follow_statement(r)
return stmts
@memoize_default(default=[])
def get_params(self):
"""
This returns the params for an Execution/Instance and is injected as a
'hack' into the parsing.Function class.
This needs to be here, because Instance can have __init__ functions,
which act the same way as normal functions.
"""
def gen_param_name_copy(param, keys=[], values=[], array_type=None):
"""
Create a param with the original scope (of varargs) as parent.
"""
parent_stmt = self.var_args.parent_stmt
pos = parent_stmt.start_pos if parent_stmt else None
calls = parsing.Array(pos, parsing.Array.NOARRAY, parent_stmt)
calls.values = values
calls.keys = keys
calls.type = array_type
new_param = copy.copy(param)
if parent_stmt is not None:
new_param.parent = parent_stmt
new_param._assignment_calls_calculated = True
new_param._assignment_calls = calls
new_param.is_generated = True
name = copy.copy(param.get_name())
name.parent = new_param
faked_scopes.append(new_param)
return name
result = []
start_offset = 0
if isinstance(self.base, InstanceElement):
# Care for self -> just exclude it and add the instance
start_offset = 1
self_name = copy.copy(self.base.params[0].get_name())
self_name.parent = self.base.instance
result.append(self_name)
param_dict = {}
for param in self.base.params:
param_dict[str(param.get_name())] = param
# There may be calls, which don't fit all the params, this just ignores
# it.
var_arg_iterator = self.get_var_args_iterator()
non_matching_keys = []
keys_used = set()
keys_only = False
for param in self.base.params[start_offset:]:
# The value and key can both be null. There, the defaults apply.
# args / kwargs will just be empty arrays / dicts, respectively.
# Wrong value count is just ignored. If you try to test cases that
# are not allowed in Python, Jedi will maybe not show any
# completions.
key, value = next(var_arg_iterator, (None, None))
while key:
keys_only = True
try:
key_param = param_dict[str(key)]
except KeyError:
non_matching_keys.append((key, value))
else:
keys_used.add(str(key))
result.append(gen_param_name_copy(key_param,
values=[value]))
key, value = next(var_arg_iterator, (None, None))
assignments = param.get_assignment_calls().values
assignment = assignments[0]
keys = []
values = []
array_type = None
if assignment[0] == '*':
# *args param
array_type = parsing.Array.TUPLE
if value:
values.append(value)
for key, value in var_arg_iterator:
# Iterate until a key argument is found.
if key:
var_arg_iterator.push_back((key, value))
break
values.append(value)
elif assignment[0] == '**':
# **kwargs param
array_type = parsing.Array.DICT
if non_matching_keys:
keys, values = zip(*non_matching_keys)
else:
# normal param
if value:
values = [value]
else:
if param.assignment_details:
# No value: return the default values.
values = assignments
else:
# If there is no assignment detail, that means there is
# no assignment, just the result. Therefore nothing has
# to be returned.
values = []
# Just ignore all the params that are without a key, after one
# keyword argument was set.
if not keys_only or assignment[0] == '**':
keys_used.add(str(key))
result.append(gen_param_name_copy(param, keys=keys,
values=values, array_type=array_type))
if keys_only:
# sometimes param arguments are not completely written (which would
# create an Exception, but we have to handle that).
for k in set(param_dict) - keys_used:
result.append(gen_param_name_copy(param_dict[k]))
return result
def get_var_args_iterator(self):
"""
Yields a key/value pair, the key is None, if its not a named arg.
"""
def iterate():
# `var_args` is typically an Array, and not a list.
for var_arg in self.var_args:
# empty var_arg
if len(var_arg) == 0:
yield None, None
# *args
elif var_arg[0] == '*':
arrays = follow_call_list([var_arg[1:]])
for array in arrays:
if hasattr(array, 'get_contents'):
for field in array.get_contents():
yield None, field
# **kwargs
elif var_arg[0] == '**':
arrays = follow_call_list([var_arg[1:]])
for array in arrays:
if hasattr(array, 'get_contents'):
for key, field in array.get_contents():
# Take the first index.
if isinstance(key, parsing.Name):
name = key
else:
# `parsing`.[Call|Function|Class] lookup.
name = key[0].name
yield name, field
# Normal arguments (including key arguments).
else:
if len(var_arg) > 1 and var_arg[1] == '=':
# This is a named parameter (var_arg[0] is a Call).
yield var_arg[0].name, var_arg[2:]
else:
yield None, var_arg
return iter(parsing.PushBackIterator(iterate()))
def get_set_vars(self):
return self.get_defined_names()
def get_defined_names(self):
"""
Call the default method with the own instance (self implements all
the necessary functions). Add also the params.
"""
return self.get_params() + parsing.Scope.get_set_vars(self)
def copy_properties(self, prop):
"""
Literally copies a property of a Function. Copying is very expensive,
because it is something like `copy.deepcopy`. However, these copied
objects can be used for the executions, as if they were in the
execution.
"""
# Copy all these lists into this local function.
attr = getattr(self.base, prop)
objects = []
for element in attr:
copied = helpers.fast_parent_copy(element)
copied.parent = self._scope_copy(copied.parent)
if isinstance(copied, parsing.Function):
copied = Function(copied)
objects.append(copied)
faked_scopes.append(copied)
return objects
def __getattr__(self, name):
if name not in ['start_pos', 'end_pos', 'imports']:
raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
return getattr(self.base, name)
@memoize_default()
def _scope_copy(self, scope):
try:
""" Copies a scope (e.g. if) in an execution """
# TODO method uses different scopes than the subscopes property.
# just check the start_pos, sometimes it's difficult with closures
# to compare the scopes directly.
if scope.start_pos == self.start_pos:
return self
else:
copied = helpers.fast_parent_copy(scope)
copied.parent = self._scope_copy(copied.parent)
faked_scopes.append(copied)
return copied
except AttributeError:
raise MultiLevelAttributeError(sys.exc_info())
@property
@memoize_default()
def returns(self):
return self.copy_properties('returns')
@property
@memoize_default()
def asserts(self):
return self.copy_properties('asserts')
@property
@memoize_default()
def statements(self):
return self.copy_properties('statements')
@property
@memoize_default()
def subscopes(self):
return self.copy_properties('subscopes')
def get_statement_for_position(self, pos):
return parsing.Scope.get_statement_for_position(self, pos)
def __repr__(self):
return "<%s of %s>" % \
(type(self).__name__, self.base)
class Generator(use_metaclass(CachedMetaClass, parsing.Base)):
""" Cares for `yield` statements. """
def __init__(self, func, var_args):
super(Generator, self).__init__()
self.func = func
self.var_args = var_args
def get_defined_names(self):
"""
Returns a list of names that define a generator, which can return the
content of a generator.
"""
names = []
none_pos = (0, 0)
executes_generator = ('__next__', 'send')
for n in ('close', 'throw') + executes_generator:
name = parsing.Name([(n, none_pos)], none_pos, none_pos)
if n in executes_generator:
name.parent = self
names.append(name)
debug.dbg('generator names', names)
return names
def iter_content(self):
""" returns the content of __iter__ """
return Execution(self.func, self.var_args).get_return_types(True)
def get_index_types(self, index=None):
debug.warning('Tried to get array access on a generator', self)
return []
@property
def parent(self):
return self.func.parent
def __repr__(self):
return "<%s of %s>" % (type(self).__name__, self.func)
class Array(use_metaclass(CachedMetaClass, parsing.Base)):
"""
Used as a mirror to parsing.Array, if needed. It defines some getter
methods which are important in this module.
"""
def __init__(self, array):
self._array = array
def get_index_types(self, index_call_list=None):
""" Get the types of a specific index or all, if not given """
# array slicing
if index_call_list is not None:
if index_call_list and [x for x in index_call_list if ':' in x]:
return [self]
index_possibilities = list(follow_call_list(index_call_list))
if len(index_possibilities) == 1:
# This is indexing only one element, with a fixed index number,
# otherwise it just ignores the index (e.g. [1+1]).
try:
# Multiple elements in the array are not wanted. var_args
# and get_only_subelement can raise AttributeErrors.
i = index_possibilities[0].var_args.get_only_subelement()
except AttributeError:
pass
else:
try:
return self.get_exact_index_types(i)
except (IndexError, KeyError):
pass
result = list(self.follow_values(self._array.values))
result += dynamic.check_array_additions(self)
return set(result)
def get_exact_index_types(self, index):
""" Here the index is an int. Raises IndexError/KeyError """
if self._array.type == parsing.Array.DICT:
old_index = index
index = None
for i, key_elements in enumerate(self._array.keys):
# Because we only want the key to be a string.
if len(key_elements) == 1:
try:
str_key = key_elements.get_code()
except AttributeError:
try:
str_key = key_elements[0].name
except AttributeError:
str_key = None
if old_index == str_key:
index = i
break
if index is None:
raise KeyError('No key found in dictionary')
values = [self._array[index]]
return self.follow_values(values)
def follow_values(self, values):
""" helper function for the index getters """
return follow_call_list(values)
def get_defined_names(self):
"""
This method generates all ArrayElements for one parsing.Array.
It returns e.g. for a list: append, pop, ...
"""
# `array.type` is a string with the type, e.g. 'list'.
scope = get_scopes_for_name(builtin.Builtin.scope, self._array.type)[0]
scope = Instance(scope)
names = scope.get_defined_names()
return [ArrayElement(n) for n in names]
def get_contents(self):
return self._array
@property
def parent(self):
"""
Return the builtin scope as parent, because the arrays are builtins
"""
return builtin.Builtin.scope
def get_parent_until(self, *args, **kwargs):
return builtin.Builtin.scope
def __getattr__(self, name):
if name not in ['type', 'start_pos']:
raise AttributeError('Strange access: %s.' % name)
return getattr(self._array, name)
def __repr__(self):
return "<e%s of %s>" % (type(self).__name__, self._array)
class ArrayElement(object):
"""
A name, e.g. `list.append`, it is used to access to original array methods.
"""
def __init__(self, name):
super(ArrayElement, self).__init__()
self.name = name
def __getattr__(self, name):
# Set access privileges:
if name not in ['parent', 'names', 'start_pos', 'end_pos', 'get_code']:
raise AttributeError('Strange access: %s.' % name)
return getattr(self.name, name)
def get_parent_until(self):
return builtin.Builtin.scope
def __repr__(self):
return "<%s of %s>" % (type(self).__name__, self.name)
def get_defined_names_for_position(scope, position=None, start_scope=None):
"""
Deletes all names that are ahead of the position, except for some special
objects like instances, where the position doesn't matter.
:param position: the position as a line/column tuple, default is infinity.
"""
names = scope.get_defined_names()
# Instances have special rules, always return all the possible completions,
# because class variables are always valid and the `self.` variables, too.
if (not position or isinstance(scope, (Array, Instance))
or start_scope != scope
and isinstance(start_scope, (parsing.Function, Execution))):
return names
names_new = []
for n in names:
if n.start_pos[0] is not None and n.start_pos < position:
names_new.append(n)
return names_new
def get_names_for_scope(scope, position=None, star_search=True,
include_builtin=True):
"""
Get all completions possible for the current scope.
The star search option is only here to provide an optimization. Otherwise
the whole thing would probably start a little recursive madness.
"""
in_func_scope = scope
non_flow = scope.get_parent_until(parsing.Flow, reverse=True,
include_current=True)
while scope:
# `parsing.Class` is used, because the parent is never `Class`.
# Ignore the Flows, because the classes and functions care for that.
# InstanceElement of Class is ignored, if it is not the start scope.
if not (scope != non_flow and scope.isinstance(parsing.Class)
or scope.isinstance(parsing.Flow)
or scope.isinstance(Instance)
and non_flow.isinstance(Function)
):
try:
if isinstance(scope, Instance):
for g in scope.scope_generator():
yield g
else:
yield scope, get_defined_names_for_position(scope,
position, in_func_scope)
except StopIteration:
raise MultiLevelStopIteration('StopIteration raised somewhere')
if scope.isinstance(parsing.ForFlow) and scope.is_list_comp:
# is a list comprehension
yield scope, scope.get_set_vars(is_internal_call=True)
scope = scope.parent
# This is used, because subscopes (Flow scopes) would distort the
# results.
if scope and scope.isinstance(Function, parsing.Function, Execution):
in_func_scope = scope
# Add star imports.
if star_search:
for s in imports.remove_star_imports(non_flow.get_parent_until()):
for g in get_names_for_scope(s, star_search=False):
yield g
# Add builtins to the global scope.
if include_builtin:
builtin_scope = builtin.Builtin.scope
yield builtin_scope, builtin_scope.get_defined_names()
def get_scopes_for_name(scope, name_str, position=None, search_global=False,
is_goto=False):
"""
This is the search function. The most important part to debug.
`remove_statements` and `filter_statements` really are the core part of
this completion.
:param position: Position of the last statement -> tuple of line, column
:return: List of Names. Their parents are the scopes, they are defined in.
:rtype: list
"""
def remove_statements(result):
"""
This is the part where statements are being stripped.
Due to lazy evaluation, statements like a = func; b = a; b() have to be
evaluated.
"""
res_new = []
for r in result:
add = []
if r.isinstance(parsing.Statement):
check_instance = None
if isinstance(r, InstanceElement) and r.is_class_var:
check_instance = r.instance
r = r.var
# Global variables handling.
if r.is_global():
for token_name in r.token_list[1:]:
if isinstance(token_name, parsing.Name):
add = get_scopes_for_name(r.parent,
str(token_name))
else:
# generated objects are used within executions, but these
# objects are in functions, and we have to dynamically
# execute first.
if isinstance(r, parsing.Param):
func = r.parent
# Instances are typically faked, if the instance is not
# called from outside. Here we check it for __init__
# functions and return.
if isinstance(func, InstanceElement) \
and func.instance.is_generated \
and hasattr(func, 'name') \
and str(func.name) == '__init__' \
and r.position_nr > 0: # 0 would be self
r = func.var.params[r.position_nr]
# add docstring knowledge
doc_params = docstrings.follow_param(r)
if doc_params:
res_new += doc_params
continue
if not r.is_generated:
res_new += dynamic.search_params(r)
if not r.assignment_details:
# this means that there are no default params,
# so just ignore it.
continue
scopes = follow_statement(r, seek_name=name_str)
add += remove_statements(scopes)
if check_instance is not None:
# class renames
add = [InstanceElement(check_instance, a, True)
if isinstance(a, (Function, parsing.Function))
else a for a in add]
res_new += add
else:
if isinstance(r, parsing.Class):
r = Class(r)
elif isinstance(r, parsing.Function):
r = Function(r)
if r.isinstance(Function):
try:
r = r.get_decorated_func()
except DecoratorNotFound:
continue
res_new.append(r)
debug.dbg('sfn remove, new: %s, old: %s' % (res_new, result))
return res_new
def filter_name(scope_generator):
"""
Filters all variables of a scope (which are defined in the
`scope_generator`), until the name fits.
"""
def handle_for_loops(loop):
# Take the first statement (for has always only
# one, remember `in`). And follow it.
if not len(loop.inits):
return []
result = get_iterator_types(follow_statement(loop.inits[0]))
if len(loop.set_vars) > 1:
var_arr = loop.set_stmt.get_assignment_calls()
result = assign_tuples(var_arr, result, name_str)
return result
def process(name):
"""
Returns the parent of a name, which means the element which stands
behind a name.
"""
result = []
no_break_scope = False
par = name.parent
if par.isinstance(parsing.Flow):
if par.command == 'for':
result += handle_for_loops(par)
else:
debug.warning('Flow: Why are you here? %s' % par.command)
elif par.isinstance(parsing.Param) \
and par.parent is not None \
and par.parent.parent.isinstance(parsing.Class) \
and par.position_nr == 0:
# This is where self gets added - this happens at another
# place, if the var_args are clear. But sometimes the class is
# not known. Therefore add a new instance for self. Otherwise
# take the existing.
if isinstance(scope, InstanceElement):
inst = scope.instance
else:
inst = Instance(Class(par.parent.parent))
inst.is_generated = True
result.append(inst)
elif par.isinstance(parsing.Statement):
def is_execution(arr):
for a in arr:
a = a[0] # rest is always empty with assignees
if a.isinstance(parsing.Array):
if is_execution(a):
return True
elif a.isinstance(parsing.Call):
if a.name == name and a.execution:
return True
return False
is_exe = False
for op, assignee in par.assignment_details:
is_exe |= is_execution(assignee)
if is_exe:
# filter array[3] = ...
# TODO check executions for dict contents
pass
else:
details = par.assignment_details
if details and details[0][0] != '=':
no_break_scope = True
# TODO this makes self variables non-breakable. wanted?
if isinstance(name, InstanceElement) \
and not name.is_class_var:
no_break_scope = True
result.append(par)
else:
result.append(par)
return result, no_break_scope
flow_scope = scope
result = []
# compare func uses the tuple of line/indent = line/column
comparison_func = lambda name: (name.start_pos)
for nscope, name_list in scope_generator:
break_scopes = []
# here is the position stuff happening (sorting of variables)
for name in sorted(name_list, key=comparison_func, reverse=True):
p = name.parent.parent if name.parent else None
if isinstance(p, InstanceElement) \
and isinstance(p.var, parsing.Class):
p = p.var
if name_str == name.get_code() and p not in break_scopes:
r, no_break_scope = process(name)
if is_goto:
if r:
# Directly assign the name, but there has to be a
# result.
result.append(name)
else:
result += r
# for comparison we need the raw class
s = nscope.base if isinstance(nscope, Class) else nscope
# this means that a definition was found and is not e.g.
# in if/else.
if result and not no_break_scope:
if not name.parent or p == s:
break
break_scopes.append(p)
while flow_scope:
# TODO check if result is in scope -> no evaluation necessary
n = dynamic.check_flow_information(flow_scope, name_str,
position)
if n:
result = n
break
if result:
break
if flow_scope == nscope:
break
flow_scope = flow_scope.parent
flow_scope = nscope
if result:
break
if not result and isinstance(nscope, Instance):
# getattr() / __getattr__ / __getattribute__
result += check_getattr(nscope, name_str)
debug.dbg('sfn filter "%s" in %s: %s' % (name_str, nscope, result))
return result
def descriptor_check(result):
""" Processes descriptors """
res_new = []
for r in result:
if isinstance(scope, (Instance, Class)) \
and hasattr(r, 'get_descriptor_return'):
# handle descriptors
try:
res_new += r.get_descriptor_return(scope)
continue
except KeyError:
pass
res_new.append(r)
return res_new
if search_global:
scope_generator = get_names_for_scope(scope, position=position)
else:
if isinstance(scope, Instance):
scope_generator = scope.scope_generator()
else:
if isinstance(scope, (Class, parsing.Module)):
# classes are only available directly via chaining?
# strange stuff...
names = scope.get_defined_names()
else:
names = get_defined_names_for_position(scope, position)
scope_generator = iter([(scope, names)])
if is_goto:
return filter_name(scope_generator)
return descriptor_check(remove_statements(filter_name(scope_generator)))
def check_getattr(inst, name_str):
result = []
# str is important to lose the NamePart!
name = parsing.Call(str(name_str), parsing.Call.STRING, (0, 0), inst)
args = helpers.generate_param_array([name])
try:
result = inst.execute_subscope_by_name('__getattr__', args)
except KeyError:
pass
if not result:
# this is a little bit special. `__getattribute__` is executed
# before anything else. But: I know no use case, where this
# could be practical and the jedi would return wrong types. If
# you ever have something, let me know!
try:
result = inst.execute_subscope_by_name('__getattribute__', args)
except KeyError:
pass
return result
def get_iterator_types(inputs):
""" Returns the types of any iterator (arrays, yields, __iter__, etc). """
iterators = []
# Take the first statement (for has always only
# one, remember `in`). And follow it.
for it in inputs:
if isinstance(it, (Generator, Array, dynamic.ArrayInstance)):
iterators.append(it)
else:
if not hasattr(it, 'execute_subscope_by_name'):
debug.warning('iterator/for loop input wrong', it)
continue
try:
iterators += it.execute_subscope_by_name('__iter__')
except KeyError:
debug.warning('iterators: No __iter__ method found.')
result = []
for gen in iterators:
if isinstance(gen, Array):
# Array is a little bit special, since this is an internal
# array, but there's also the list builtin, which is
# another thing.
result += gen.get_index_types()
elif isinstance(gen, Instance):
# __iter__ returned an instance.
name = '__next__' if is_py3k else 'next'
try:
result += gen.execute_subscope_by_name(name)
except KeyError:
debug.warning('Instance has no __next__ function', gen)
else:
# is a generator
result += gen.iter_content()
return result
def assign_tuples(tup, results, seek_name):
"""
This is a normal assignment checker. In python functions and other things
can return tuples:
>>> a, b = 1, ""
>>> a, (b, c) = 1, ("", 1.0)
Here, if seek_name is "a", the number type will be returned.
The first part (before `=`) is the param tuples, the second one result.
:type tup: parsing.Array
"""
def eval_results(index):
types = []
for r in results:
if hasattr(r, "get_exact_index_types"):
try:
types += r.get_exact_index_types(index)
except IndexError:
pass
else:
debug.warning("invalid tuple lookup %s of result %s in %s"
% (tup, results, seek_name))
return types
result = []
if tup.type == parsing.Array.NOARRAY:
# Here we have unnessecary braces, which we just remove.
arr = tup.get_only_subelement()
if type(arr) == parsing.Call:
if arr.name.names[-1] == seek_name:
result = results
else:
result = assign_tuples(arr, results, seek_name)
else:
for i, t in enumerate(tup):
# Used in assignments. There is just one call and no other things,
# therefore we can just assume, that the first part is important.
if len(t) != 1:
raise AttributeError('Array length should be 1')
t = t[0]
# Check the left part, if there are still tuples in it or a Call.
if isinstance(t, parsing.Array):
# These are "sub"-tuples.
result += assign_tuples(t, eval_results(i), seek_name)
else:
if t.name.names[-1] == seek_name:
result += eval_results(i)
return result
@helpers.RecursionDecorator
@memoize_default(default=[])
def follow_statement(stmt, seek_name=None):
"""
The starting point of the completion. A statement always owns a call list,
which are the calls, that a statement does.
In case multiple names are defined in the statement, `seek_name` returns
the result for this name.
:param stmt: A `parsing.Statement`.
:param seek_name: A string.
"""
debug.dbg('follow_stmt %s (%s)' % (stmt, seek_name))
call_list = stmt.get_assignment_calls()
debug.dbg('calls: %s' % call_list)
try:
result = follow_call_list(call_list)
except AttributeError:
# This is so evil! But necessary to propagate errors. The attribute
# errors here must not be catched, because they shouldn't exist.
raise MultiLevelAttributeError(sys.exc_info())
# Assignment checking is only important if the statement defines multiple
# variables.
if len(stmt.get_set_vars()) > 1 and seek_name and stmt.assignment_details:
new_result = []
for op, set_vars in stmt.assignment_details:
new_result += assign_tuples(set_vars, result, seek_name)
result = new_result
return set(result)
def follow_call_list(call_list):
"""
The call_list has a special structure.
This can be either `parsing.Array` or `list of list`.
It is used to evaluate a two dimensional object, that has calls, arrays and
operators in it.
"""
def evaluate_list_comprehension(lc, parent=None):
input = lc.input
nested_lc = lc.input.token_list[0]
if isinstance(nested_lc, parsing.ListComprehension):
# is nested LC
input = nested_lc.stmt
loop = parsing.ForFlow([input], lc.stmt.start_pos,
lc.middle, True)
if parent is None:
loop.parent = lc.stmt.parent
else:
loop.parent = parent
if isinstance(nested_lc, parsing.ListComprehension):
loop = evaluate_list_comprehension(nested_lc, loop)
return loop
if parsing.Array.is_type(call_list, parsing.Array.TUPLE,
parsing.Array.DICT):
# Tuples can stand just alone without any braces. These would be
# recognized as separate calls, but actually are a tuple.
result = follow_call(call_list)
else:
result = []
for calls in call_list:
calls_iterator = iter(calls)
for call in calls_iterator:
if parsing.Array.is_type(call, parsing.Array.NOARRAY):
result += follow_call_list(call)
elif isinstance(call, parsing.ListComprehension):
loop = evaluate_list_comprehension(call)
stmt = copy.copy(call.stmt)
stmt.parent = loop
# create a for loop which does the same as list
# comprehensions
result += follow_statement(stmt)
else:
# With things like params, these can also be functions...
if isinstance(call, (Function, Class, Instance,
dynamic.ArrayInstance)):
result.append(call)
# The string tokens are just operations (+, -, etc.)
elif not isinstance(call, (str, unicode)):
if str(call.name) == 'if':
# Ternary operators.
while True:
try:
call = next(calls_iterator)
except StopIteration:
break
try:
if str(call.name) == 'else':
break
except AttributeError:
pass
continue
result += follow_call(call)
elif call == '*':
if [r for r in result if isinstance(r, Array)
or isinstance(r, Instance)
and str(r.name) == 'str']:
# if it is an iterable, ignore * operations
next(calls_iterator)
return set(result)
def follow_call(call):
""" Follow a call is following a function, variable, string, etc. """
scope = call.parent_stmt.parent
path = call.generate_call_path()
position = call.parent_stmt.start_pos
return follow_call_path(path, scope, position)
def follow_call_path(path, scope, position):
""" Follows a path generated by `parsing.Call.generate_call_path()` """
current = next(path)
if isinstance(current, parsing.Array):
result = [Array(current)]
else:
if not isinstance(current, parsing.NamePart):
if current.type in (parsing.Call.STRING, parsing.Call.NUMBER):
t = type(current.name).__name__
scopes = get_scopes_for_name(builtin.Builtin.scope, t)
else:
debug.warning('unknown type:', current.type, current)
scopes = []
# Make instances of those number/string objects.
arr = helpers.generate_param_array([current.name])
scopes = [Instance(s, arr) for s in scopes]
else:
# This is the first global lookup.
scopes = get_scopes_for_name(scope, current, position=position,
search_global=True)
result = imports.strip_imports(scopes)
return follow_paths(path, result, scope, position=position)
def follow_paths(path, results, call_scope, position=None):
"""
In each result, `path` must be followed. Copies the path iterator.
"""
results_new = []
if results:
if len(results) > 1:
iter_paths = itertools.tee(path, len(results))
else:
iter_paths = [path]
for i, r in enumerate(results):
fp = follow_path(iter_paths[i], r, call_scope, position=position)
if fp is not None:
results_new += fp
else:
# This means stop iteration.
return results
return results_new
def follow_path(path, scope, call_scope, position=None):
"""
Uses a generator and tries to complete the path, e.g.
>>> foo.bar.baz
`follow_path` is only responsible for completing `.bar.baz`, the rest is
done in the `follow_call` function.
"""
# Current is either an Array or a Scope.
try:
current = next(path)
except StopIteration:
return None
debug.dbg('follow %s in scope %s' % (current, scope))
result = []
if isinstance(current, parsing.Array):
# This must be an execution, either () or [].
if current.type == parsing.Array.LIST:
if hasattr(scope, 'get_index_types'):
result = scope.get_index_types(current)
elif current.type not in [parsing.Array.DICT]:
# Scope must be a class or func - make an instance or execution.
debug.dbg('exe', scope)
result = Execution(scope, current).get_return_types()
else:
# Curly braces are not allowed, because they make no sense.
debug.warning('strange function call with {}', current, scope)
else:
# The function must not be decorated with something else.
if scope.isinstance(Function):
scope = scope.get_magic_method_scope()
else:
# This is the typical lookup while chaining things.
if filter_private_variable(scope, call_scope, current):
return []
result = imports.strip_imports(get_scopes_for_name(scope, current,
position=position))
return follow_paths(path, set(result), call_scope, position=position)
def filter_private_variable(scope, call_scope, var_name):
if isinstance(var_name, (str, unicode)) \
and var_name.startswith('__') and isinstance(scope, Instance):
s = call_scope.get_parent_until((parsing.Class, Instance),
include_current=True)
if s != scope and s != scope.base.base:
return True
return False
def goto(stmt, call_path=None):
if call_path is None:
arr = stmt.get_assignment_calls()
call = arr.get_only_subelement()
call_path = list(call.generate_call_path())
scope = stmt.parent
pos = stmt.start_pos
call_path, search = call_path[:-1], call_path[-1]
if call_path:
scopes = follow_call_path(iter(call_path), scope, pos)
search_global = False
pos = None
else:
scopes = [scope]
search_global = True
follow_res = []
for s in scopes:
follow_res += get_scopes_for_name(s, search, pos,
search_global=search_global, is_goto=True)
return follow_res, search