341 lines
12 KiB
Python
341 lines
12 KiB
Python
|
from copy import deepcopy
|
||
|
|
from itertools import product
|
||
|
|
|
||
|
|
from pythonforandroid.logger import info
|
||
|
|
from pythonforandroid.recipe import Recipe
|
||
|
|
from pythonforandroid.bootstrap import Bootstrap
|
||
|
|
from pythonforandroid.util import BuildInterruptingException
|
||
|
|
|
||
|
|
|
||
|
|
def fix_deplist(deps):
|
||
|
|
""" Turn a dependency list into lowercase, and make sure all entries
|
||
|
|
that are just a string become a tuple of strings
|
||
|
|
"""
|
||
|
|
deps = [
|
||
|
|
((dep.lower(),)
|
||
|
|
if not isinstance(dep, (list, tuple))
|
||
|
|
else tuple([dep_entry.lower()
|
||
|
|
for dep_entry in dep
|
||
|
|
]))
|
||
|
|
for dep in deps
|
||
|
|
]
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
class RecipeOrder(dict):
|
||
|
|
def __init__(self, ctx):
|
||
|
|
self.ctx = ctx
|
||
|
|
|
||
|
|
def conflicts(self):
|
||
|
|
for name in self.keys():
|
||
|
|
try:
|
||
|
|
recipe = Recipe.get_recipe(name, self.ctx)
|
||
|
|
conflicts = [dep.lower() for dep in recipe.conflicts]
|
||
|
|
except ValueError:
|
||
|
|
conflicts = []
|
||
|
|
|
||
|
|
if any([c in self for c in conflicts]):
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def get_dependency_tuple_list_for_recipe(recipe, blacklist=None):
|
||
|
|
""" Get the dependencies of a recipe with filtered out blacklist, and
|
||
|
|
turned into tuples with fix_deplist()
|
||
|
|
"""
|
||
|
|
if blacklist is None:
|
||
|
|
blacklist = set()
|
||
|
|
assert(type(blacklist) == set)
|
||
|
|
if recipe.depends is None:
|
||
|
|
dependencies = []
|
||
|
|
else:
|
||
|
|
# Turn all dependencies into tuples so that product will work
|
||
|
|
dependencies = fix_deplist(recipe.depends)
|
||
|
|
|
||
|
|
# Filter out blacklisted items and turn lowercase:
|
||
|
|
dependencies = [
|
||
|
|
tuple(set(deptuple) - blacklist)
|
||
|
|
for deptuple in dependencies
|
||
|
|
if tuple(set(deptuple) - blacklist)
|
||
|
|
]
|
||
|
|
return dependencies
|
||
|
|
|
||
|
|
|
||
|
|
def recursively_collect_orders(
|
||
|
|
name, ctx, all_inputs, orders=None, blacklist=None
|
||
|
|
):
|
||
|
|
'''For each possible recipe ordering, try to add the new recipe name
|
||
|
|
to that order. Recursively do the same thing with all the
|
||
|
|
dependencies of each recipe.
|
||
|
|
|
||
|
|
'''
|
||
|
|
name = name.lower()
|
||
|
|
if orders is None:
|
||
|
|
orders = []
|
||
|
|
if blacklist is None:
|
||
|
|
blacklist = set()
|
||
|
|
try:
|
||
|
|
recipe = Recipe.get_recipe(name, ctx)
|
||
|
|
dependencies = get_dependency_tuple_list_for_recipe(
|
||
|
|
recipe, blacklist=blacklist
|
||
|
|
)
|
||
|
|
|
||
|
|
# handle opt_depends: these impose requirements on the build
|
||
|
|
# order only if already present in the list of recipes to build
|
||
|
|
dependencies.extend(fix_deplist(
|
||
|
|
[[d] for d in recipe.get_opt_depends_in_list(all_inputs)
|
||
|
|
if d.lower() not in blacklist]
|
||
|
|
))
|
||
|
|
|
||
|
|
if recipe.conflicts is None:
|
||
|
|
conflicts = []
|
||
|
|
else:
|
||
|
|
conflicts = [dep.lower() for dep in recipe.conflicts]
|
||
|
|
except ValueError:
|
||
|
|
# The recipe does not exist, so we assume it can be installed
|
||
|
|
# via pip with no extra dependencies
|
||
|
|
dependencies = []
|
||
|
|
conflicts = []
|
||
|
|
|
||
|
|
new_orders = []
|
||
|
|
# for each existing recipe order, see if we can add the new recipe name
|
||
|
|
for order in orders:
|
||
|
|
if name in order:
|
||
|
|
new_orders.append(deepcopy(order))
|
||
|
|
continue
|
||
|
|
if order.conflicts():
|
||
|
|
continue
|
||
|
|
if any([conflict in order for conflict in conflicts]):
|
||
|
|
continue
|
||
|
|
|
||
|
|
for dependency_set in product(*dependencies):
|
||
|
|
new_order = deepcopy(order)
|
||
|
|
new_order[name] = set(dependency_set)
|
||
|
|
|
||
|
|
dependency_new_orders = [new_order]
|
||
|
|
for dependency in dependency_set:
|
||
|
|
dependency_new_orders = recursively_collect_orders(
|
||
|
|
dependency, ctx, all_inputs, dependency_new_orders,
|
||
|
|
blacklist=blacklist
|
||
|
|
)
|
||
|
|
|
||
|
|
new_orders.extend(dependency_new_orders)
|
||
|
|
|
||
|
|
return new_orders
|
||
|
|
|
||
|
|
|
||
|
|
def find_order(graph):
|
||
|
|
'''
|
||
|
|
Do a topological sort on the dependency graph dict.
|
||
|
|
'''
|
||
|
|
while graph:
|
||
|
|
# Find all items without a parent
|
||
|
|
leftmost = [l for l, s in graph.items() if not s]
|
||
|
|
if not leftmost:
|
||
|
|
raise ValueError('Dependency cycle detected! %s' % graph)
|
||
|
|
# If there is more than one, sort them for predictable order
|
||
|
|
leftmost.sort()
|
||
|
|
for result in leftmost:
|
||
|
|
# Yield and remove them from the graph
|
||
|
|
yield result
|
||
|
|
graph.pop(result)
|
||
|
|
for bset in graph.values():
|
||
|
|
bset.discard(result)
|
||
|
|
|
||
|
|
|
||
|
|
def obvious_conflict_checker(ctx, name_tuples, blacklist=None):
|
||
|
|
""" This is a pre-flight check function that will completely ignore
|
||
|
|
recipe order or choosing an actual value in any of the multiple
|
||
|
|
choice tuples/dependencies, and just do a very basic obvious
|
||
|
|
conflict check.
|
||
|
|
"""
|
||
|
|
deps_were_added_by = dict()
|
||
|
|
deps = set()
|
||
|
|
if blacklist is None:
|
||
|
|
blacklist = set()
|
||
|
|
|
||
|
|
# Add dependencies for all recipes:
|
||
|
|
to_be_added = [(name_tuple, None) for name_tuple in name_tuples]
|
||
|
|
while len(to_be_added) > 0:
|
||
|
|
current_to_be_added = list(to_be_added)
|
||
|
|
to_be_added = []
|
||
|
|
for (added_tuple, adding_recipe) in current_to_be_added:
|
||
|
|
assert(type(added_tuple) == tuple)
|
||
|
|
if len(added_tuple) > 1:
|
||
|
|
# No obvious commitment in what to add, don't check it itself
|
||
|
|
# but throw it into deps for later comparing against
|
||
|
|
# (Remember this function only catches obvious issues)
|
||
|
|
deps.add(added_tuple)
|
||
|
|
continue
|
||
|
|
|
||
|
|
name = added_tuple[0]
|
||
|
|
recipe_conflicts = set()
|
||
|
|
recipe_dependencies = []
|
||
|
|
try:
|
||
|
|
# Get recipe to add and who's ultimately adding it:
|
||
|
|
recipe = Recipe.get_recipe(name, ctx)
|
||
|
|
recipe_conflicts = {c.lower() for c in recipe.conflicts}
|
||
|
|
recipe_dependencies = get_dependency_tuple_list_for_recipe(
|
||
|
|
recipe, blacklist=blacklist
|
||
|
|
)
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
adder_first_recipe_name = adding_recipe or name
|
||
|
|
|
||
|
|
# Collect the conflicts:
|
||
|
|
triggered_conflicts = []
|
||
|
|
for dep_tuple_list in deps:
|
||
|
|
# See if the new deps conflict with things added before:
|
||
|
|
if set(dep_tuple_list).intersection(
|
||
|
|
recipe_conflicts) == set(dep_tuple_list):
|
||
|
|
triggered_conflicts.append(dep_tuple_list)
|
||
|
|
continue
|
||
|
|
|
||
|
|
# See if what was added before conflicts with the new deps:
|
||
|
|
if len(dep_tuple_list) > 1:
|
||
|
|
# Not an obvious commitment to a specific recipe/dep
|
||
|
|
# to be added, so we won't check.
|
||
|
|
# (remember this function only catches obvious issues)
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
dep_recipe = Recipe.get_recipe(dep_tuple_list[0], ctx)
|
||
|
|
except ValueError:
|
||
|
|
continue
|
||
|
|
conflicts = [c.lower() for c in dep_recipe.conflicts]
|
||
|
|
if name in conflicts:
|
||
|
|
triggered_conflicts.append(dep_tuple_list)
|
||
|
|
|
||
|
|
# Throw error on conflict:
|
||
|
|
if triggered_conflicts:
|
||
|
|
# Get first conflict and see who added that one:
|
||
|
|
adder_second_recipe_name = "'||'".join(triggered_conflicts[0])
|
||
|
|
second_recipe_original_adder = deps_were_added_by.get(
|
||
|
|
(adder_second_recipe_name,), None
|
||
|
|
)
|
||
|
|
if second_recipe_original_adder:
|
||
|
|
adder_second_recipe_name = second_recipe_original_adder
|
||
|
|
|
||
|
|
# Prompt error:
|
||
|
|
raise BuildInterruptingException(
|
||
|
|
"Conflict detected: '{}'"
|
||
|
|
" inducing dependencies {}, and '{}'"
|
||
|
|
" inducing conflicting dependencies {}".format(
|
||
|
|
adder_first_recipe_name,
|
||
|
|
(recipe.name,),
|
||
|
|
adder_second_recipe_name,
|
||
|
|
triggered_conflicts[0]
|
||
|
|
))
|
||
|
|
|
||
|
|
# Actually add it to our list:
|
||
|
|
deps.add(added_tuple)
|
||
|
|
deps_were_added_by[added_tuple] = adding_recipe
|
||
|
|
|
||
|
|
# Schedule dependencies to be added
|
||
|
|
to_be_added += [
|
||
|
|
(dep, adder_first_recipe_name or name)
|
||
|
|
for dep in recipe_dependencies
|
||
|
|
if dep not in deps
|
||
|
|
]
|
||
|
|
# If we came here, then there were no obvious conflicts.
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None):
|
||
|
|
# Get set of recipe/dependency names, clean up and add bootstrap deps:
|
||
|
|
names = set(names)
|
||
|
|
if bs is not None and bs.recipe_depends:
|
||
|
|
names = names.union(set(bs.recipe_depends))
|
||
|
|
names = fix_deplist([
|
||
|
|
([name] if not isinstance(name, (list, tuple)) else name)
|
||
|
|
for name in names
|
||
|
|
])
|
||
|
|
if blacklist is None:
|
||
|
|
blacklist = set()
|
||
|
|
blacklist = {bitem.lower() for bitem in blacklist}
|
||
|
|
|
||
|
|
# Remove all values that are in the blacklist:
|
||
|
|
names_before_blacklist = list(names)
|
||
|
|
names = []
|
||
|
|
for name in names_before_blacklist:
|
||
|
|
cleaned_up_tuple = tuple([
|
||
|
|
item for item in name if item not in blacklist
|
||
|
|
])
|
||
|
|
if cleaned_up_tuple:
|
||
|
|
names.append(cleaned_up_tuple)
|
||
|
|
|
||
|
|
# Do check for obvious conflicts (that would trigger in any order, and
|
||
|
|
# without comitting to any specific choice in a multi-choice tuple of
|
||
|
|
# dependencies):
|
||
|
|
obvious_conflict_checker(ctx, names, blacklist=blacklist)
|
||
|
|
# If we get here, no obvious conflicts!
|
||
|
|
|
||
|
|
# get all possible order graphs, as names may include tuples/lists
|
||
|
|
# of alternative dependencies
|
||
|
|
possible_orders = []
|
||
|
|
for name_set in product(*names):
|
||
|
|
new_possible_orders = [RecipeOrder(ctx)]
|
||
|
|
for name in name_set:
|
||
|
|
new_possible_orders = recursively_collect_orders(
|
||
|
|
name, ctx, name_set, orders=new_possible_orders,
|
||
|
|
blacklist=blacklist
|
||
|
|
)
|
||
|
|
possible_orders.extend(new_possible_orders)
|
||
|
|
|
||
|
|
# turn each order graph into a linear list if possible
|
||
|
|
orders = []
|
||
|
|
for possible_order in possible_orders:
|
||
|
|
try:
|
||
|
|
order = find_order(possible_order)
|
||
|
|
except ValueError: # a circular dependency was found
|
||
|
|
info('Circular dependency found in graph {}, skipping it.'.format(
|
||
|
|
possible_order))
|
||
|
|
continue
|
||
|
|
orders.append(list(order))
|
||
|
|
|
||
|
|
# prefer python3 and SDL2 if available
|
||
|
|
orders.sort(key=lambda order: -('python3' in order) - ('sdl2' in order))
|
||
|
|
|
||
|
|
if not orders:
|
||
|
|
raise BuildInterruptingException(
|
||
|
|
'Didn\'t find any valid dependency graphs. '
|
||
|
|
'This means that some of your '
|
||
|
|
'requirements pull in conflicting dependencies.')
|
||
|
|
|
||
|
|
# It would be better to check against possible orders other
|
||
|
|
# than the first one, but in practice clashes will be rare,
|
||
|
|
# and can be resolved by specifying more parameters
|
||
|
|
chosen_order = orders[0]
|
||
|
|
if len(orders) > 1:
|
||
|
|
info('Found multiple valid dependency orders:')
|
||
|
|
for order in orders:
|
||
|
|
info(' {}'.format(order))
|
||
|
|
info('Using the first of these: {}'.format(chosen_order))
|
||
|
|
else:
|
||
|
|
info('Found a single valid recipe set: {}'.format(chosen_order))
|
||
|
|
|
||
|
|
if bs is None:
|
||
|
|
bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx)
|
||
|
|
if bs is None:
|
||
|
|
# Note: don't remove this without thought, causes infinite loop
|
||
|
|
raise BuildInterruptingException(
|
||
|
|
"Could not find any compatible bootstrap!"
|
||
|
|
)
|
||
|
|
recipes, python_modules, bs = get_recipe_order_and_bootstrap(
|
||
|
|
ctx, chosen_order, bs=bs, blacklist=blacklist
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
# check if each requirement has a recipe
|
||
|
|
recipes = []
|
||
|
|
python_modules = []
|
||
|
|
for name in chosen_order:
|
||
|
|
try:
|
||
|
|
recipe = Recipe.get_recipe(name, ctx)
|
||
|
|
python_modules += recipe.python_depends
|
||
|
|
except ValueError:
|
||
|
|
python_modules.append(name)
|
||
|
|
else:
|
||
|
|
recipes.append(name)
|
||
|
|
|
||
|
|
python_modules = list(set(python_modules))
|
||
|
|
return recipes, python_modules, bs
|