WIP : embedded query into mutate_deep_narrow_path,

added function variable_substitution_deep_narrow_mut_query

edited minor changes - gp_learner.py, gp_query.py

edited requirement.txt

Author: nandikaK

config/defaults.py

@@ -82,5 +82,9 @@
 MUTPB_DN_MIN_LEN = 2  # minimum length of the deep and narrow paths
 MUTPB_DN_MAX_LEN = 10  # absolute max of path length if not stopped by term_pb
 MUTPB_DN_TERM_PB = 0.3  # prob to terminate node expansion each step > min_len
+MUTPB_DN_FILTER_NODE_COUNT = 10
+MUTPB_DN_FILTER_EDGE_COUNT = 1
+MUTPB_DN_QUERY_LIMIT = 32
 # for import in helpers and __init__
+
 __all__ = [_v for _v in globals().keys() if _v.isupper()]

gp_learner.py

@@ -42,6 +42,7 @@
 from cluster import select_best_variant
 import config
 from gp_query import ask_multi_query
+from gp_query import variable_substitution_deep_narrow_mut_query
 from gp_query import calibrate_query_timeout
 from gp_query import combined_ask_count_multi_query
 from gp_query import predict_query
@@ -420,7 +421,7 @@ def _mutate_expand_node_helper(node, pb_en_out_link=config.MUTPB_EN_OUT_LINK):
         new_triple = (node, var_edge, var_node)
     else:
         new_triple = (var_node, var_edge, node)
-    return new_triple, var_node
+    return new_triple, var_node, var_edge
 
 
 def mutate_expand_node(child, node=None):
@@ -433,11 +434,10 @@ def mutate_expand_node(child, node=None):
 
 
 def mutate_deep_narrow_path(
-        child,
+        child, sparql, timeout, gtp_scores,
         min_len=config.MUTPB_DN_MIN_LEN,
         max_len=config.MUTPB_DN_MAX_LEN,
         term_pb=config.MUTPB_DN_TERM_PB,
-        pb_en_out_link=config.MUTPB_EN_OUT_LINK,
 ):
     assert isinstance(child, GraphPattern)
     nodes = list(child.nodes)
@@ -451,15 +451,76 @@ def mutate_deep_narrow_path(
         if hop >= max_len:
             break
         hop += 1
-        new_triple, var_node = _mutate_expand_node_helper(start_node)
-        gp += [new_triple]
-        start_node = var_node
+        new_triple, var_node, var_edge = _mutate_expand_node_helper(start_node)
+        test_gp = gp + [new_triple]
+        test_gp, fixed = _mutate_deep_narrow_path_helper(
+            sparql, timeout, gtp_scores, test_gp, var_edge, var_node)
+        if fixed:
+            start_node = var_node
+            gp = test_gp
 
     # TODO: insert connection to a target node
     # TODO: fix edge or node ( to_count_var_over_values_query)
     return gp
 
 
+def _mutate_deep_narrow_path_helper(
+            sparql,
+            timeout,
+            gtp_scores,
+            child,
+            edge_var,
+            node_var,
+            gtp_sample_n=config.MUTPB_FV_RGTP_SAMPLE_N,
+            limit_res=config.MUTPB_DN_QUERY_LIMIT,
+            sample_n=config.MUTPB_FV_SAMPLE_MAXN,
+):
+    assert isinstance(child, GraphPattern)
+    assert isinstance(gtp_scores, GTPScores)
+
+    # The further we get, the less gtps are remaining. Sampling too many (all)
+    # of them might hurt as common substitutions (> limit ones) which are dead
+    # ends could cover less common ones that could actually help
+    gtp_sample_n = min(gtp_sample_n, int(gtp_scores.remaining_gain))
+    gtp_sample_n = random.randint(1, gtp_sample_n)
+
+    ground_truth_pairs = gtp_scores.remaining_gain_sample_gtps(
+     n=gtp_sample_n)
+    t, substitution_counts = variable_substitution_deep_narrow_mut_query(
+     sparql, timeout, child, edge_var, node_var, ground_truth_pairs,
+     limit_res)
+    if not substitution_counts:
+        # the current pattern is unfit, as we can't find anything fulfilling it
+        logger.debug("tried to fix a var %s without result:\n%s"
+                      "seems as if the pattern can't be fulfilled!",
+                      edge_var, child.to_sparql_select_query())
+        fixed = False
+        return [child], fixed
+    mutate_fix_var_filter(substitution_counts)
+    if not substitution_counts:
+        # could have happened that we removed the only possible substitution
+        fixed = False
+        return [child], fixed
+    # randomly pick n of the substitutions with a prob ~ to their counts
+    items, counts = zip(*substitution_counts.most_common())
+    substs = sample_from_list(items, counts, sample_n)
+    logger.info(
+     'fixed variable %s in %sto:\n %s\n<%d out of:\n%s\n',
+     edge_var.n3(),
+     child,
+     '\n '.join([subst.n3() for subst in substs]),
+     sample_n,
+     '\n'.join([' %d: %s' % (c, v.n3())
+                for v, c in substitution_counts.most_common()]),
+    )
+    fixed = True
+    res = [
+     GraphPattern(child, mapping={edge_var: subst})
+     for subst in substs
+     ]
+    return res, fixed
+
+
 def mutate_add_edge(child):
     # TODO: can maybe be improved by sparqling
     nodes = list(child.nodes)
@@ -682,6 +743,7 @@ def mutate(
         pb_dt=config.MUTPB_DT,
         pb_en=config.MUTPB_EN,
         pb_fv=config.MUTPB_FV,
+        pb_dn=config.MUTPB_DN,
         pb_id=config.MUTPB_ID,
         pb_iv=config.MUTPB_IV,
         pb_mv=config.MUTPB_MV,
@@ -721,6 +783,9 @@ def mutate(
     if random.random() < pb_sp:
         child = mutate_simplify_pattern(child)
 
+    if random.random() < pb_dn:
+        child = mutate_deep_narrow_path(child, sparql, timeout, gtp_scores)
+
     if random.random() < pb_fv:
         child = canonicalize(child)
         children = mutate_fix_var(sparql, timeout, gtp_scores, child)

gp_query.py

@@ -426,6 +426,21 @@ def variable_substitution_query(
     )
 
 
+def variable_substitution_deep_narrow_mut_query(
+        sparql, timeout, graph_pattern, edge_var, node_var,
+        source_target_pairs, limit_res, batch_size=config.BATCH_SIZE):
+    _vars, _values, _ret_val_mapping = _get_vars_values_mapping(
+        graph_pattern, source_target_pairs)
+    _edge_var_node_var_and_vars = (edge_var, node_var, _vars)
+    return _multi_query(
+        sparql, timeout, graph_pattern, source_target_pairs, batch_size,
+        _edge_var_node_var_and_vars, _values, _ret_val_mapping,
+        _var_subst_res_init, _var_subst_dnp_chunk_q,
+        _var_subst_dnp_chunk_result_ext, limit=limit_res,
+        # non standard, passed via **kwds, see handling below
+    )
+
+
 # noinspection PyUnusedLocal
 def _var_subst_res_init(_, **kwds):
     return Counter()
@@ -440,6 +455,17 @@ def _var_subst_chunk_q(gp, _sel_var_and_vars, values_chunk, limit):
         limit=limit)
 
 
+def _var_subst_dnp_chunk_q(gp, _edge_var_node_var_and_vars,
+                           values_chunk, limit):
+    edge_var, node_var, _vars = _edge_var_node_var_and_vars
+    return gp.to_find_edge_var_for_narrow_path_query(
+        edge_var=edge_var,
+        node_var=node_var,
+        vars_=_vars,
+        values={_vars: values_chunk},
+        limit_res=limit)
+
+
 # noinspection PyUnusedLocal
 def _var_subst_chunk_result_ext(q_res, _sel_var_and_vars, _, **kwds):
     var, _vars = _sel_var_and_vars
@@ -456,6 +482,21 @@ def _var_subst_chunk_result_ext(q_res, _sel_var_and_vars, _, **kwds):
     return chunk_res
 
 
+def _var_subst_dnp_chunk_result_ext(q_res, _edge_var_node_var_and_vars, _, **kwds):
+    edge_var, node_var, _vars = _edge_var_node_var_and_vars
+    chunk_res = Counter()
+    res_rows_path = ['results', 'bindings']
+    bindings = sparql_json_result_bindings_to_rdflib(
+        get_path(q_res, res_rows_path, default=[])
+    )
+
+    for row in bindings:
+        row_res = get_path(row, [edge_var])
+        count_res = int(get_path(row, [COUNT_VAR], '0'))
+        chunk_res[row_res] += count_res
+    return chunk_res
+
+
 def _var_subst_res_update(res, update, **_):
     res += update
 

graph_pattern.py

@@ -31,6 +31,7 @@
 import six
 
 from utils import URIShortener
+import config
 
 logger = logging.getLogger(__name__)
 
@@ -41,10 +42,12 @@
 TARGET_VAR = Variable('target')
 ASK_VAR = Variable('ask')
 COUNT_VAR = Variable('count')
-EDGE_VAR_COUNT = Variable('edge_count_var')
-NODE_VAR_COUNT = Variable('node_count_var')
-MAX_NODE_COUNT = Variable('maximum node count')
-PRIO_VAR = Variable('priority')
+EDGE_VAR_COUNT = Variable('edge_var_count')
+NODE_VAR_COUNT = Variable('node_var_count')
+MAX_NODE_COUNT = Variable('max_node_count')
+PRIO_VAR = Variable('prio_var')
+
+
 def gen_random_var():
     return Variable(RANDOM_VAR_PREFIX + ''.join(
         random.choice(string.ascii_letters + string.digits)
@@ -711,9 +714,10 @@ def to_count_var_over_values_query(self, var, vars_, values, limit):
         res += 'LIMIT %d\n' % limit
         return self._sparql_prefix(res)
 
-    def to_find_edge_var_for_narrow_path_query(self, edge_var, node_var,
-                                               vars_, filter_node_count,
-                                               filter_edge_count, limit_res):
+    def to_find_edge_var_for_narrow_path_query\
+                    (self, edge_var, node_var, vars_, values, limit_res,
+                     filter_node_count=config.MUTPB_DN_FILTER_NODE_COUNT,
+                     filter_edge_count=config.MUTPB_DN_FILTER_EDGE_COUNT):
         """Counts possible substitutions for edge_var to get a narrow path
 
         Meant to perform a query like this:
@@ -759,21 +763,16 @@ def to_find_edge_var_for_narrow_path_query(self, edge_var, node_var,
 
         res = 'SELECT * WHERE {\n'
         res += ' {\n'\
-               '  SELECT %s (COUNT(*) as %s) (Max(%s) AS %s) ' \
-               '   (COUNT(*)/AVG(%s) AS %s) WHERE {\n' % (
+               '  SELECT %s (COUNT(*) AS %s) (MAX(%s) AS %s) ' \
+               '(COUNT(*)/AVG(%s) AS %s) WHERE {\n' % (
                      edge_var.n3(), EDGE_VAR_COUNT.n3(),
                      NODE_VAR_COUNT.n3(), MAX_NODE_COUNT.n3(),
                      NODE_VAR_COUNT.n3(), PRIO_VAR.n3())
         res += '    SELECT DISTINCT %s %s (COUNT(%s) AS %s) WHERE {\n' % (
                    ' '.join([v.n3() for v in vars_]),
                    edge_var.n3(), node_var.n3(), NODE_VAR_COUNT.n3())
-        # res += self._sparql_values_part(values)
-        res += 'VALUES(%s) {\n' \
-               '(dbr: Adolescence dbr: Youth)' \
-               '(dbr:Adult dbr:Child)' \
-               '(dbr:Angel  dbr:Heaven)' \
-               '(dbr:Arithmetic dbr:Mathematics)' \
-               '}\n' % (' '.join([v.n3() for v in vars_]))
+        res += self._sparql_values_part(values)
+
         # triples part
         tres = []
         for s, p, o in self:

requirements.txt

@@ -5,11 +5,11 @@ matplotlib==1.5.1
 networkx==1.11
 nose==1.3.7
 numpy==1.11.0
-# pygraphviz==1.3.1
+pygraphviz==1.3.1
 requests==2.9.1
 rdflib==4.2.1
 scikit-learn==0.17.1
-scipy
+scipy==0.17.0
 scoop==0.7.1.1
 six==1.10.0
 SPARQLWrapper==1.7.6