mutate_deep_narrow_path() is expanding the graph pattern, and fixing suitable edges

Author: nandikaK

gp_learner.py

@@ -429,7 +429,7 @@ def mutate_expand_node(child, node=None):
     if not node:
         nodes = list(child.nodes)
         node = random.choice(nodes)
-    new_triple, _ = _mutate_expand_node_helper(node)
+    new_triple, _, _ = _mutate_expand_node_helper(node)
     return child + (new_triple,)
 
 
@@ -452,15 +452,10 @@ def mutate_deep_narrow_path(
             break
         hop += 1
         new_triple, var_node, var_edge = _mutate_expand_node_helper(start_node)
-        test_gp = gp + [new_triple]
-        test_gp, fixed = _mutate_deep_narrow_path_helper(
-            sparql, timeout, gtp_scores, test_gp, var_edge, var_node)
-        if fixed:
-            start_node = var_node
-            gp = test_gp
-
-    # TODO: insert connection to a target node
-    # TODO: fix edge or node ( to_count_var_over_values_query)
+        gp += [new_triple]
+        gp, fixed = _mutate_deep_narrow_path_helper(
+            sparql, timeout, gtp_scores, gp, var_edge, var_node)
+        start_node = var_node
     return gp
 
 
@@ -489,36 +484,52 @@ def _mutate_deep_narrow_path_helper(
     t, substitution_counts = variable_substitution_deep_narrow_mut_query(
      sparql, timeout, child, edge_var, node_var, ground_truth_pairs,
      limit_res)
-    if not substitution_counts:
+    edge_count, node_sum_count = substitution_counts
+    if not node_sum_count:
         # the current pattern is unfit, as we can't find anything fulfilling it
         logger.debug("tried to fix a var %s without result:\n%s"
-                      "seems as if the pattern can't be fulfilled!",
-                      edge_var, child.to_sparql_select_query())
+                     "seems as if the pattern can't be fulfilled!",
+                     edge_var, child.to_sparql_select_query())
         fixed = False
-        return [child], fixed
-    mutate_fix_var_filter(substitution_counts)
-    if not substitution_counts:
+        return child, fixed
+    mutate_fix_var_filter(node_sum_count)
+    mutate_fix_var_filter(edge_count)
+    if not node_sum_count:
         # could have happened that we removed the only possible substitution
         fixed = False
-        return [child], fixed
+        return child, fixed
+
+    prio = Counter()
+    for edge, node_sum in node_sum_count.items():
+        ec = edge_count[edge]
+        prio[edge] = ec / (node_sum / ec)  # ec / AVG degree
     # randomly pick n of the substitutions with a prob ~ to their counts
-    items, counts = zip(*substitution_counts.most_common())
-    substs = sample_from_list(items, counts, sample_n)
+    edges, prios = zip(*prio.most_common())
+
+    substs = sample_from_list(edges, prios, sample_n)
+
     logger.info(
-     'fixed variable %s in %sto:\n %s\n<%d out of:\n%s\n',
-     edge_var.n3(),
-     child,
-     '\n '.join([subst.n3() for subst in substs]),
-     sample_n,
-     '\n'.join([' %d: %s' % (c, v.n3())
-                for v, c in substitution_counts.most_common()]),
+        'fixed variable %s in %sto:\n %s\n<%d out of:\n%s\n',
+        edge_var.n3(),
+        child,
+        '\n '.join([subst.n3() for subst in substs]),
+        sample_n,
+        '\n'.join([
+            ' %.3f: %s' % (c, v.n3()) for v, c in prio.most_common()]),
     )
     fixed = True
-    res = [
+    orig_child = child
+    children = [
      GraphPattern(child, mapping={edge_var: subst})
      for subst in substs
      ]
-    return res, fixed
+    children = [
+        c if fit_to_live(c) else orig_child
+        for c in children
+        ]
+    if children:
+        child = random.choice(list(children))
+    return child, fixed
 
 
 def mutate_add_edge(child):

gp_query.py

@@ -32,6 +32,8 @@
 from graph_pattern import TARGET_VAR
 from graph_pattern import ASK_VAR
 from graph_pattern import COUNT_VAR
+from graph_pattern import NODE_VAR_SUM
+from graph_pattern import EDGE_VAR_COUNT
 from utils import chunker
 from utils import exception_stack_catcher
 from utils import get_path
@@ -282,7 +284,6 @@ def _combined_chunk_res(q_res, _vars, _ret_val_mapping):
     return chunk_res
 
 
-
 def count_query(sparql, timeout, graph_pattern, source=None,
                 **kwds):
     assert isinstance(graph_pattern, GraphPattern)
@@ -426,21 +427,6 @@ def variable_substitution_query(
     )
 
 
-def variable_substitution_deep_narrow_mut_query(
-        sparql, timeout, graph_pattern, edge_var, node_var,
-        source_target_pairs, limit_res, batch_size=config.BATCH_SIZE):
-    _vars, _values, _ret_val_mapping = _get_vars_values_mapping(
-        graph_pattern, source_target_pairs)
-    _edge_var_node_var_and_vars = (edge_var, node_var, _vars)
-    return _multi_query(
-        sparql, timeout, graph_pattern, source_target_pairs, batch_size,
-        _edge_var_node_var_and_vars, _values, _ret_val_mapping,
-        _var_subst_res_init, _var_subst_dnp_chunk_q,
-        _var_subst_dnp_chunk_result_ext, limit=limit_res,
-        # non standard, passed via **kwds, see handling below
-    )
-
-
 # noinspection PyUnusedLocal
 def _var_subst_res_init(_, **kwds):
     return Counter()
@@ -455,17 +441,6 @@ def _var_subst_chunk_q(gp, _sel_var_and_vars, values_chunk, limit):
         limit=limit)
 
 
-def _var_subst_dnp_chunk_q(gp, _edge_var_node_var_and_vars,
-                           values_chunk, limit):
-    edge_var, node_var, _vars = _edge_var_node_var_and_vars
-    return gp.to_find_edge_var_for_narrow_path_query(
-        edge_var=edge_var,
-        node_var=node_var,
-        vars_=_vars,
-        values={_vars: values_chunk},
-        limit_res=limit)
-
-
 # noinspection PyUnusedLocal
 def _var_subst_chunk_result_ext(q_res, _sel_var_and_vars, _, **kwds):
     var, _vars = _sel_var_and_vars
@@ -482,23 +457,70 @@ def _var_subst_chunk_result_ext(q_res, _sel_var_and_vars, _, **kwds):
     return chunk_res
 
 
-def _var_subst_dnp_chunk_result_ext(q_res, _edge_var_node_var_and_vars, _, **kwds):
+def _var_subst_res_update(res, update, **_):
+    res += update
+
+
+def variable_substitution_deep_narrow_mut_query(
+        sparql, timeout, graph_pattern, edge_var, node_var,
+        source_target_pairs, limit_res, batch_size=config.BATCH_SIZE):
+    _vars, _values, _ret_val_mapping = _get_vars_values_mapping(
+        graph_pattern, source_target_pairs)
+    _edge_var_node_var_and_vars = (edge_var, node_var, _vars)
+    return _multi_query(
+        sparql, timeout, graph_pattern, source_target_pairs, batch_size,
+        _edge_var_node_var_and_vars, _values, _ret_val_mapping,
+        _var_subst_dnp_res_init, _var_subst_dnp_chunk_q,
+        _var_subst_dnp_chunk_result_ext,
+        _res_update=_var_subst_dnp_update,
+        limit=limit_res,
+        # non standard, passed via **kwds, see handling below
+    )
+
+
+# noinspection PyUnusedLocal
+def _var_subst_dnp_res_init(_, **kwds):
+    return Counter(), Counter()
+
+
+def _var_subst_dnp_chunk_q(gp, _edge_var_node_var_and_vars,
+                           values_chunk, limit):
     edge_var, node_var, _vars = _edge_var_node_var_and_vars
-    chunk_res = Counter()
+    return gp.to_find_edge_var_for_narrow_path_query(
+        edge_var=edge_var,
+        node_var=node_var,
+        vars_=_vars,
+        values={_vars: values_chunk},
+        limit_res=limit)
+
+
+# noinspection PyUnusedLocal
+def _var_subst_dnp_chunk_result_ext(
+        q_res, _edge_var_node_var_and_vars, _, **kwds):
+    edge_var, node_var, _vars = _edge_var_node_var_and_vars
+    chunk_edge_count, chunk_node_sum = Counter(), Counter()
     res_rows_path = ['results', 'bindings']
     bindings = sparql_json_result_bindings_to_rdflib(
         get_path(q_res, res_rows_path, default=[])
     )
 
     for row in bindings:
         row_res = get_path(row, [edge_var])
-        count_res = int(get_path(row, [COUNT_VAR], '0'))
-        chunk_res[row_res] += count_res
-    return chunk_res
+        edge_count = int(get_path(row, [EDGE_VAR_COUNT], '0'))
+        chunk_edge_count[row_res] += edge_count
+        node_sum_count = int(get_path(row, [NODE_VAR_SUM], '0'))
+        chunk_node_sum[row_res] += node_sum_count
+    return chunk_edge_count, chunk_node_sum,
 
 
-def _var_subst_res_update(res, update, **_):
-    res += update
+def _var_subst_dnp_update(res, up, **_):
+    edge_count, node_sum_count = res
+    try:
+        chunk_edge_count, chunk_node_sum = up
+        edge_count.update(chunk_edge_count)
+        node_sum_count.update(chunk_node_sum)
+    except ValueError:
+        pass
 
 
 def generate_stps_from_gp(sparql, gp):

graph_pattern.py

@@ -43,9 +43,7 @@
 ASK_VAR = Variable('ask')
 COUNT_VAR = Variable('count')
 EDGE_VAR_COUNT = Variable('edge_var_count')
-NODE_VAR_COUNT = Variable('node_var_count')
-MAX_NODE_COUNT = Variable('max_node_count')
-PRIO_VAR = Variable('prio_var')
+NODE_VAR_SUM = Variable('node_var_sum')
 
 
 def gen_random_var():
@@ -581,7 +579,7 @@ def _sparql_query_pattern_part(
         res = self._sparql_values_part(values, indent)
         tres = []
         for s, p, o in self:
-            tres.append('%s %s %s .' % (s.n3(), p.n3(), o.n3()))
+            tres.append('%s %s %s.' % (s.n3(), p.n3(), o.n3()))
         res += indent + ('\n' + indent).join(tres) + '\n'
         if bind:
             res += '%sFILTER(\n' % indent
@@ -667,22 +665,7 @@ def to_count_var_over_values_query(self, var, vars_, values, limit):
         """Counts possible fulfilling substitutions for var.
 
         Meant to perform a query like this:
-         SELECT ?var count(*) as ?count WHERE {
-          VALUES (?source ?target) {
-           (dbr:Adolescence dbr:Youth)
-           (dbr:Adult dbr:Child)
-           (dbr:Angel dbr:Heaven)
-           (dbr:Arithmetic dbr:Mathematics)
-          }
-          {
-           SELECT DISTINCT ?source ?target ?var WHERE {
-            ?source ?edge ?target .
-            ?var dbo:wikiPageWikiLink ?target .
-           }
-          }
-         }
-         ORDER BY desc(?count)
-         LIMIT 10
+
 
         :param var: Variable to count over.
         :param vars_: List of vars to fix values for (e.g. ?source, ?target).
@@ -714,10 +697,11 @@ def to_count_var_over_values_query(self, var, vars_, values, limit):
         res += 'LIMIT %d\n' % limit
         return self._sparql_prefix(res)
 
-    def to_find_edge_var_for_narrow_path_query\
-                    (self, edge_var, node_var, vars_, values, limit_res,
-                     filter_node_count=config.MUTPB_DN_FILTER_NODE_COUNT,
-                     filter_edge_count=config.MUTPB_DN_FILTER_EDGE_COUNT):
+    def to_find_edge_var_for_narrow_path_query(
+            self, edge_var, node_var, vars_, values, limit_res,
+            filter_node_count=config.MUTPB_DN_FILTER_NODE_COUNT,
+            filter_edge_count=config.MUTPB_DN_FILTER_EDGE_COUNT,
+    ):
         """Counts possible substitutions for edge_var to get a narrow path
 
         Meant to perform a query like this:
@@ -763,14 +747,15 @@ def to_find_edge_var_for_narrow_path_query\
 
         res = 'SELECT * WHERE {\n'
         res += ' {\n'\
-               '  SELECT %s (COUNT(*) AS %s) (MAX(%s) AS %s) ' \
-               '(COUNT(*)/AVG(%s) AS %s) WHERE {\n' % (
-                     edge_var.n3(), EDGE_VAR_COUNT.n3(),
-                     NODE_VAR_COUNT.n3(), MAX_NODE_COUNT.n3(),
-                     NODE_VAR_COUNT.n3(), PRIO_VAR.n3())
-        res += '    SELECT DISTINCT %s %s (COUNT(%s) AS %s) WHERE {\n' % (
-                   ' '.join([v.n3() for v in vars_]),
-                   edge_var.n3(), node_var.n3(), NODE_VAR_COUNT.n3())
+               '  SELECT %s (SUM (?node_var_count) AS %s) (COUNT(%s) AS %s) ' \
+               '(MAX(?node_var_count) AS ?max_node_count) WHERE {\n' % (
+                     edge_var.n3(),
+                     NODE_VAR_SUM.n3(),
+                     ' && '.join([v.n3() for v in vars_]),
+                     EDGE_VAR_COUNT.n3(), )
+        res += '    SELECT DISTINCT %s %s (COUNT(%s) AS ?node_var_count) ' \
+               'WHERE {\n   ' % (' '.join([v.n3() for v in vars_]),
+                                 edge_var.n3(), node_var.n3(), )
         res += self._sparql_values_part(values)
 
         # triples part
@@ -783,14 +768,12 @@ def to_find_edge_var_for_narrow_path_query\
         res += '    }\n'\
                '   }\n'
         res += '   GROUP BY %s\n' % edge_var.n3()
-        res += '   ORDER BY DESC(%s)\n' % EDGE_VAR_COUNT.n3()
         res += '  }\n'
-        res += '  FILTER(%s < %d && %s > %d)\n' % (MAX_NODE_COUNT.n3(),
-                                                 filter_node_count,
-                                                 EDGE_VAR_COUNT.n3(),
-                                                 filter_edge_count)
+        res += '  FILTER(?max_node_count < %d && %s > %d)\n' \
+               % (filter_node_count, EDGE_VAR_COUNT.n3(),
+                  filter_edge_count)
         res += '}\n'
-        res += 'ORDER BY DESC(%s)\n' % PRIO_VAR.n3()
+        res += 'ORDER BY ASC(%s)\n' % NODE_VAR_SUM.n3()
         res += 'LIMIT %d' % limit_res
         return self._sparql_prefix(res)