[MRQL-97] Support for source-level debugging
diff --git a/core/src/main/java/org/apache/mrql/Bag.java b/core/src/main/java/org/apache/mrql/Bag.java
index 33424d0..24dd966 100644
--- a/core/src/main/java/org/apache/mrql/Bag.java
+++ b/core/src/main/java/org/apache/mrql/Bag.java
@@ -87,7 +87,8 @@
         mode = Modes.STREAMED;
         iterator = i;
         consumed = false;
-        if (Config.debug)
+        // during debugging, bags are materialized to vectors
+        if (Config.debug || Config.lineage)
             materialize();
     }
 
diff --git a/core/src/main/java/org/apache/mrql/Debugger.gen b/core/src/main/java/org/apache/mrql/Debugger.gen
index 266594d..de6267a 100644
--- a/core/src/main/java/org/apache/mrql/Debugger.gen
+++ b/core/src/main/java/org/apache/mrql/Debugger.gen
@@ -18,7 +18,6 @@
 package org.apache.mrql;
 
 import org.apache.mrql.gen.*;
-import java.util.Enumeration;
 import java.io.*;
 import javax.swing.*;
 import javax.swing.JTree.*;
@@ -33,7 +32,9 @@
     final MRData result;
     final JTree tree;
     final JTextField search = new JTextField(20);
+    static JFrame frame;
     private static volatile boolean exit = false;
+    private static volatile boolean trace_nodes_only = false;
     Trees exprs;
 
     /** wrapped strings to be colored red */
@@ -50,6 +51,38 @@
         JToolBar toolBar = new JToolBar("");
         toolBar.setPreferredSize(new Dimension(1000,40));
         JButton button = new JButton();
+        button.setActionCommand("Exit");
+        button.setText("exit");
+        final JFrame nframe = frame;
+        button.addActionListener(new ActionListener() {
+                public void actionPerformed ( ActionEvent e ) {
+                    nframe.setVisible(false);
+                    nframe.dispose();
+                    exit = true;
+                }
+            });
+        button.setVisible(true);
+        toolBar.add(button);
+        toolBar.add(Box.createRigidArea(new Dimension(200,0)));
+        JCheckBox checkb = new JCheckBox();
+        checkb.setActionCommand("traceOnly");
+        checkb.setText("trace nodes only");
+        checkb.addActionListener(new ActionListener() {
+                public void actionPerformed ( ActionEvent e ) {
+                    trace_nodes_only = !trace_nodes_only;
+                    if (trace_nodes_only) {
+                        search.setText("");
+                        DefaultTreeModel model = (DefaultTreeModel)tree.getModel();
+                        DefaultMutableTreeNode root = createNode(result,"","");
+                        root.setUserObject("results with trace and input nodes only");
+                        model.setRoot(root);
+                        model.reload(root);
+                    } else reset();
+                }
+            });
+        checkb.setVisible(true);
+        toolBar.add(checkb);
+        button = new JButton();
         button.setActionCommand("Previous");
         button.setText("prev");
         button.addActionListener(new ActionListener() {
@@ -57,7 +90,7 @@
                 }
             });
         button.setVisible(false);
-        toolBar.add(button);
+        //toolBar.add(button);
         button = new JButton();
         button.setActionCommand("Next");
         button.setText("next");
@@ -66,19 +99,14 @@
                 }
             });
         button.setVisible(false);
-        toolBar.add(button);
-        toolBar.add(Box.createRigidArea(new Dimension(400,0)));
+        //toolBar.add(button);
+        toolBar.add(Box.createRigidArea(new Dimension(200,0)));
         button = new JButton();
         button.setActionCommand("Clear");
         button.setText("clear");
         button.addActionListener(new ActionListener() {
                 public void actionPerformed ( ActionEvent e ) {
-                    search.setText("");
-                    DefaultTreeModel model = (DefaultTreeModel)tree.getModel();
-                    DefaultMutableTreeNode root = createNode(result,"","");
-                    root.setUserObject("results");
-                    model.setRoot(root);
-                    model.reload(root);
+                    reset();
                 }
             });
         toolBar.add(button);
@@ -146,10 +174,21 @@
 
     public void valueChanged ( TreeSelectionEvent e ) { }
 
+    private void reset () {
+        search.setText("");
+        DefaultTreeModel model = (DefaultTreeModel)tree.getModel();
+        DefaultMutableTreeNode root = createNode(result,"","");
+        root.setUserObject("results");
+        model.setRoot(root);
+        model.reload(root);
+    }
+
     private String exprNode ( Tree e ) {
         match e {
         case call(source,_,`path,...):
-            return "source: "+path;
+            return path.stringValue();
+        case trace(`msg,_,_):
+            return msg.stringValue();
         case call(`f,...):
             return f.toString();
         case cmap(lambda(`v,_),_):
@@ -170,78 +209,87 @@
         return e.toString();
     }
 
-    private DefaultMutableTreeNode create_node ( MRData value, int n, String inputSearch ) {
-        if (value instanceof Tuple) {
-            DefaultMutableTreeNode node = new DefaultMutableTreeNode("input "+n);
-            DefaultMutableTreeNode child = provenanceNode(value,inputSearch);
-            if (child.getUserObject() instanceof TaggedString)
-                node.setUserObject(new TaggedString("input "+n));
-            node.add(child);
-            return node;
-        } else if (value instanceof Bag) {
-            DefaultMutableTreeNode node = new DefaultMutableTreeNode("input "+n);
-            boolean matched = false;
-            for ( MRData e: (Bag)value ) {
-                DefaultMutableTreeNode child = provenanceNode(e,inputSearch);
-                matched |= child.getUserObject() instanceof TaggedString;
-                node.add(child);
-            };
-            if (matched)
-                node.setUserObject(new TaggedString("input "+n));
-            return node;
-        } else return new DefaultMutableTreeNode(value.toString());
+    private boolean consider_node ( Tree e ) {
+        match e {
+        case call(source,...): return true;
+        case trace(...): return true;
+        };
+        return !trace_nodes_only;
     }
 
-    private DefaultMutableTreeNode provenanceNode ( MRData value, String inputSearch ) {
+    private boolean existing_child ( DefaultMutableTreeNode node, DefaultMutableTreeNode parent ) {
+        Object no = node.getUserObject();
+        String ns = (no instanceof TaggedString) ? ((TaggedString)no).value : (String)no;
+        for ( int i = 0; i < parent.getChildCount(); i++ ) {
+            Object co = ((DefaultMutableTreeNode)parent.getChildAt(i)).getUserObject();
+            String cs = (co instanceof TaggedString) ? ((TaggedString)co).value : (String)co;
+            if (cs.equals(ns))
+                return true;
+        };
+        return false;
+    }
+
+    private void create_nodes ( MRData value, DefaultMutableTreeNode parent,
+                                String inputSearch, boolean first ) {
         Tuple p = ((Tuple)value);
         MRData v = p.get(1);
-        DefaultMutableTreeNode node = new DefaultMutableTreeNode(v.toString());
         Tree opr = exprs.nth(((MR_int)p.get(0)).get());
-        node.add(new DefaultMutableTreeNode(exprNode(opr)));
+        Tree tp = exprs.nth(((MR_int)p.get(0)).get()-1);
+        match tp {
+        case `T(`etp):
+            if (!Provenance.is_collection(T))
+                fail;
+            tp = etp;
+        }
+        DefaultMutableTreeNode node
+            = new DefaultMutableTreeNode((first) ? Printer.print(v,tp)
+                                         : exprNode(opr)+": "+Printer.print(v,tp));
+        if (first || consider_node(opr)) {
+            if (!existing_child(node,parent))
+                parent.add(node);
+        } else node = parent;
+        for ( int i = 2; i < p.size(); i++ )
+            if (p.get(i) instanceof Bag)
+                for ( MRData e: (Bag)p.get(i) )
+                    create_nodes(e,node,inputSearch,false);
+            else create_nodes(p.get(i),node,inputSearch,false);
         boolean matched = false;
-        for ( int i = 2; i < p.size(); i++ ) {
-            DefaultMutableTreeNode child = create_node(p.get(i),i-1,inputSearch);
-            node.add(child);
+        for ( int i = 0; i < node.getChildCount(); i++ ) {
+            DefaultMutableTreeNode child = (DefaultMutableTreeNode)node.getChildAt(i);
             matched |= child.getUserObject() instanceof TaggedString;
         };
-        if (matched)
-            node.setUserObject(new TaggedString(v.toString()));
+        if (matched && node.getUserObject() instanceof String)
+            node.setUserObject(new TaggedString((String)node.getUserObject()));
         else match opr {
         case call(source,...):
-            if (!inputSearch.equals("") && v.toString().contains(inputSearch))
-                node.setUserObject(new TaggedString(v.toString()));
-        };
-        return node;
+            if (!inputSearch.equals("") && v.toString().contains(inputSearch)
+                && node.getUserObject() instanceof String)
+                node.setUserObject(new TaggedString((String)node.getUserObject()));
+        }
     }
 
     private DefaultMutableTreeNode createNode ( MRData value, String outputSearch, String inputSearch ) {
         if (value instanceof Bag) {
             DefaultMutableTreeNode node = new DefaultMutableTreeNode("results");
             for ( MRData e: (Bag)value )
-                if (outputSearch.equals("") || ((Tuple)e).get(0).toString().contains(outputSearch)) {
-                    DefaultMutableTreeNode child = provenanceNode(((Tuple)e).get(1),inputSearch);
-                    if (inputSearch.equals("") || (child.getUserObject() instanceof TaggedString))
-                        node.add(child);
-                };
+                if (outputSearch.equals("") || ((Tuple)e).get(0).toString().contains(outputSearch))
+                    create_nodes(((Tuple)e).get(1),node,inputSearch,true);
             return node;
         } else if (value instanceof MR_dataset) {
             DefaultMutableTreeNode node = new DefaultMutableTreeNode("results");
             for ( MRData e: ((MR_dataset)value).dataset().take(Config.max_bag_size_print) )
-                if (outputSearch.equals("") || ((Tuple)e).get(0).toString().contains(outputSearch)) {
-                    DefaultMutableTreeNode child = provenanceNode(((Tuple)e).get(1),inputSearch);
-                    if (inputSearch.equals("") || (child.getUserObject() instanceof TaggedString))
-                        node.add(child);
-                };
+                if (outputSearch.equals("") || ((Tuple)e).get(0).toString().contains(outputSearch))
+                    create_nodes(((Tuple)e).get(1),node,inputSearch,true);
             return node;
         } else {
-            DefaultMutableTreeNode node = new DefaultMutableTreeNode(((Tuple)value).get(0).toString());
-            node.add(provenanceNode(((Tuple)value).get(1),inputSearch));
+            DefaultMutableTreeNode node = new DefaultMutableTreeNode("value");
+            if (outputSearch.equals("") || ((Tuple)value).get(0).toString().contains(outputSearch))
+                create_nodes(((Tuple)value).get(1),node,inputSearch,true);
             return node;
         }
     }
-    
+
     private static void createAndShowGUI ( MRData lineage, Tree tp, Trees exprs ) {
-        final JFrame frame = new JFrame("MRQL debugger");
         frame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE);
  	frame.addWindowListener(new WindowListener() {
                 public void windowClosing ( WindowEvent e ) {
@@ -268,12 +316,14 @@
      */
     public static void debug ( final MRData lineage, final Tree type, final Trees exprs ) {
         try {
+            exit = false;
+            trace_nodes_only = false;
+            frame = new JFrame("MRQL debugger");
             SwingUtilities.invokeLater(new Runnable() {
                     public void run () {
                         createAndShowGUI(lineage,type,exprs);
                     }
                 });
-            exit = false;
             while (!exit)
                 Thread.sleep(1000);
         } catch (Exception ex) {
diff --git a/core/src/main/java/org/apache/mrql/Interpreter.gen b/core/src/main/java/org/apache/mrql/Interpreter.gen
index 37e2147..c138c47 100644
--- a/core/src/main/java/org/apache/mrql/Interpreter.gen
+++ b/core/src/main/java/org/apache/mrql/Interpreter.gen
@@ -243,6 +243,7 @@
     }
 
     public static MRData trace ( long count, Tree type, MRData value ) {
+        value.materializeAll();
         System.out.print(tabs(tab_count)+"--> "+count+": ");
         System.out.println(Printer.print(value,type));
         tab_count -= 3;
@@ -813,6 +814,10 @@
                     });
             case let(`v,`u,`body):
                 return evalM(body,new Environment(v.toString(),evalE(u,env),env));
+            case materialize(`u):
+                Bag res = evalM(u,env);
+                res.materialize();
+                return res;
             case trace(`msg,`tp,`x):
                 String m = ((MR_string)evalE(msg,env)).get();
                 return (Bag)trace(pre_trace(m),tp,evalM(x,env));
diff --git a/core/src/main/java/org/apache/mrql/Provenance.gen b/core/src/main/java/org/apache/mrql/Provenance.gen
index 62cae72..0ae272c 100644
--- a/core/src/main/java/org/apache/mrql/Provenance.gen
+++ b/core/src/main/java/org/apache/mrql/Provenance.gen
@@ -66,6 +66,8 @@
      * @return a provenance tuple
      */
     private static Tree prov ( Tree expr, Tree value, Trees provenance ) {
+        Tree tp = TypeInference.type_inference(expr);
+        exprs = exprs.append(tp);
         exprs = exprs.append(expr);
         int loc = exprs.length()-1;
         if (value.is_variable())
@@ -81,7 +83,7 @@
     private static Tree lift_var ( Tree var, Tree nvar, Tree fvar, Tree e ) {
         match e {
         case cmap(`f,`x):
-            if (fine_grain)
+            if (fine_grain || contains_trace(f))
                 fail;
             // don't lift the cmap function in coarse-grained provenance
             Tree nf = subst(var,fvar,f);
@@ -96,19 +98,30 @@
         return (e.equals(var)) ? nvar : e;
     }
 
+    private static boolean contains_trace ( Tree e ) {
+        match e {
+        case trace(`msg,`tp,`x):
+            return true;
+        case `f(...as):
+            for ( Tree a: as )
+                if (contains_trace(a))
+                    return true;
+        };
+        return false;
+    }
+
     /** Lift the expression e of type {t} to {(t,{provenance})} */
     private static Tree embedB ( Tree e ) {
         match e {
         case repeat(lambda(`v,`u),`x,`n):
             Tree nv = new_var();
-            Tree nn = new_var();
             Tree ex = embedB(x);
-            Tree ef = lift_var(v,nv,#<cmap(lambda(`nn,bag(nth(`nn,0))),`nv)>,
-                               flipr(embedB(u)));
+            Tree ef = subst(v,nv,flipr(embedB(u)));
             return #<repeat(lambda(`nv,`ef),`ex,`n)>;
         case cmap(lambda(`v,`b),`x):
-            if (fine_grain)
+            if (fine_grain || contains_trace(b))
                 fail;
+            // coarse grain
             Tree nv = new_var();
             Tree nw = new_var();
             Tree ex = embedB(x);
@@ -116,10 +129,11 @@
             Tree p = prov(e,nw,#<nth(`nv,1)>);
             return #<cmap(lambda(`nv,cmap(lambda(`nw,bag(`p)),`nb)),`ex)>;
         case cmap(lambda(`v,`b),`x):
+            // fine grain
             Tree nv = new_var();
             Tree nw = new_var();
             Tree ex = embedB(x);
-            Tree ef = lift_var(v,nv,#<nth(`nv,0)>,embedB(b));
+            Tree ef = subst(v,nv,embedB(b));
             Tree p = prov(e,#<nth(`nw,0)>,#<nth(`nw,1)>);
             return #<cmap(lambda(`nv,cmap(lambda(`nw,bag(`p)),`ef)),`ex)>;
         case groupBy(`x):
@@ -142,8 +156,8 @@
             Tree ey = flip(embedB(y));
             Tree val = #<tuple(nth(`nv,0),tuple(`(first(xv)),`(first(yv))))>;
             Tree p = prov(e,val,#[`(second(xv)),`(second(yv))]);
-            return #<cmap(lambda(`nv,bag(let(`xv,nth(nth(`nv,1),0),
-                                             let(`yv,nth(nth(`nv,1),1),`p)))),
+            return #<cmap(lambda(`nv,let(`xv,nth(nth(`nv,1),0),
+                                         let(`yv,nth(nth(`nv,1),1),bag(`p)))),
                           coGroup(`ex,`ey))>;
         case call(source,...):
             Tree nv = new_var();
@@ -170,11 +184,25 @@
                                       project(nth(`nw,0),`a)))>;
         case if(`pred,`x,`y):
             Tree nv = new_var();
+            Tree nw = new_var();
             Tree ep = embedP(pred);
             Tree ex = embedB(x);
             Tree ey = embedB(y);
-            return #<let(`nv,tuple(`ep,`ex,`ey),
-                         if(nth(nth(`nv,0),0),nth(`nv,1),nth(`nv,2)))>;
+            Tree p = prov(e,#<nth(`nw,0)>,#[nth(`nv,1),nth(`nw,1)]);
+            return #<let(`nv,`ep,
+                         cmap(lambda(`nw,bag(`p)),if(nth(`nv,0),`ex,`ey)))>;
+        case typed(`x,`T(`tp)):
+            if (!is_collection(T))
+                fail;
+            Tree nv = new_var();
+            Tree ex = embedB(x);
+            Tree p = prov(e,#<typed(nth(`nv,0),`tp)>,#<nth(`nv,1)>);
+            return #<cmap(lambda(`nv,bag(`p)),`ex)>;
+        case trace(`msg,`tp,`x):
+            Tree nv = new_var();
+            Tree ex = embedB(x);
+            Tree p = prov(e,#<nth(`nv,0)>,#<nth(`nv,1)>);
+            return #<cmap(lambda(`nv,bag(`p)),`ex)>;
         case `v:
             if (v.is_variable())
                 if (Interpreter.repeat_variables.member(v))
@@ -213,7 +241,6 @@
             Tree ex = embedB(x);
             Tree p = prov(e,#<reduce(`m,`(first(nv)))>,second(nv));
             return #<let(`nv,`ex,`p)>;
-            //return #<Let(`nv,`ex,`p)>;
         case tuple(...as):
             Tree nv = new_var();
             Trees es = #[ ];
@@ -276,13 +303,45 @@
             Tree p = prov(e,#<if(nth(nth(`nv,0),0),nth(nth(`nv,1),0),nth(nth(`nv,2),0))>,
                           #[nth(nth(`nv,0),1),nth(nth(`nv,1),1),nth(nth(`nv,2),1)]);
             return #<let(`nv,tuple(`ep,`ex,`ey),`p)>;
-        case typed(`u,_):
-            return embedP(u);
         case index(`x,`n):
             Tree ex = embedB(x);
             return #<index(`ex,`n)>;
         case true: return prov(e,e,#[ ]);
         case false: return prov(e,e,#[ ]);
+        case typed(tagged_union(`n,`u),`tp):
+            Tree nv = new_var();
+            Tree ex = embedP(u);
+            Tree p = prov(e,#<typed(tagged_union(`n,nth(`nv,0)),`tp)>,#<nth(`nv,1)>);
+            return #<let(`nv,`ex,`p)>;
+        case typed(union_value(`u),`tp):
+            Tree nv = new_var();
+            Tree ex = embedP(u);
+            Tree p = prov(e,#<typed(union_value(nth(`nv,0)),`tp)>,#<nth(`nv,1)>);
+            return #<let(`nv,`ex,`p)>;
+        case typed(`x,`tp):
+            Tree nv = new_var();
+            Tree ex = embedP(x);
+            Tree p = prov(e,#<typed(nth(`nv,0),`tp)>,#<nth(`nv,1)>);
+            return #<let(`nv,`ex,`p)>;
+        case trace(`msg,`tp,`x):
+            Tree nv = new_var();
+            Tree ex = embedP(x);
+            Tree p = prov(e,#<nth(`nv,0)>,#<nth(`nv,1)>);
+            return #<let(`nv,`ex,`p)>;
+        case `f(...as):
+            Tree nv = new_var();
+            Trees es = #[ ];
+            Trees vs = #[ ];
+            Trees ps = #[ ];
+            int i = 0;
+            for ( Tree a: as ) {
+                es = es.append(embedP(a));
+                vs = vs.append(#<nth(nth(`nv,`i),0)>);
+                ps = ps.append(#<nth(nth(`nv,`i),1)>);
+                i++;
+            };
+            Tree p = prov(e,#<`f(...vs)>,ps);
+            return #<let(`nv,tuple(...es),`p)>;
         case `v:
             if (v.is_variable())
                 if (Interpreter.lookup_global_binding(v.toString()) != null)
@@ -305,6 +364,7 @@
             if (!is_collection(T))
                 fail;
             ne = SimplifyTerm(embedB(ne));
+            TypeInference.type_inference(ne);
         case _: ne = SimplifyTerm(embedP(ne));
         };
         ne = SimplifyTerm(convert_to_algebra(ne));
@@ -319,29 +379,40 @@
         return false;
     }
 
-    /** Collect the provenance leaves (the data sources that contribute to the output) into a bag */
-    private static Bag collect_lineage ( MRData value ) {
+    /** Print the provenance trace and leaf nodes
+     *   (the data sources that have contributed to the output value) */
+    private static void print_lineage ( MRData value, boolean trace_only, String tab ) {
         if (value instanceof Tuple) {
             Tuple p = ((Tuple)value);
-            if (p.size() == 2)
-                match exprs.nth(((MR_int)p.get(0)).get()) {
-                case call(source,...):
-                    return new Bag(p.get(1));
-                };
-            Bag s = new Bag();
-            for ( int i = 2; i < p.size() && s.size() < Config.max_bag_size_print; i++ )
-                for ( MRData e: collect_lineage(p.get(i)) )
-                    s.add(e);
-            return s;
+            Tree e = exprs.nth(((MR_int)p.get(0)).get());
+            match e {
+            case call(source,_,`path,...):
+                if (trace_only)
+                    fail;
+                match exprs.nth(((MR_int)p.get(0)).get()-1) {
+                case `T(`tp):
+                    if (!is_collection(T))
+                        fail;
+                    System.out.println(tab+path.stringValue()+": "
+                                       +Printer.print(p.get(1),tp));
+                }
+            case trace(`msg,`T(`tp),`x):
+                if (!is_collection(T))
+                    fail;
+                System.out.println(tab+msg.stringValue()+": "+Printer.print(p.get(1),tp));
+                print_lineage(p.get(2),true,tab+"    ");
+            case trace(`msg,`tp,`x):
+                System.out.println(tab+msg.stringValue()+": "+Printer.print(p.get(1),tp));
+                print_lineage(p.get(2),true,tab+"    ");
+            case _:
+                for ( int i = 2; i < p.size(); i++ )
+                    print_lineage(p.get(i),trace_only,tab);
+            }
         } else if (value instanceof Bag) {
-            Bag s = new Bag();
             ((Bag)value).materialize();
             for ( MRData e: (Bag)value )
-                for ( MRData x: collect_lineage(e) )
-                    if (s.size() < Config.max_bag_size_print)
-                        s.add(x);
-            return s;
-        } else return new Bag();
+                print_lineage(e,trace_only,tab);
+        }
     }
 
     /** Print the data sources that contribute to the output */
@@ -354,15 +425,15 @@
             if (value instanceof Bag)
                 for ( MRData e: (Bag)value ) {
                     System.out.println(Printer.print(((Tuple)e).get(0),etp));
-                    System.out.println(" <- "+collect_lineage(((Tuple)e).get(1)));
+                    print_lineage(((Tuple)e).get(1),false,"    ");
                 } else if (value instanceof MR_dataset)
                 for ( MRData e: ((MR_dataset)value).dataset().take(Config.max_bag_size_print) ) {
                     System.out.println(Printer.print(((Tuple)e).get(0),etp));
-                    System.out.println(" <- "+collect_lineage(((Tuple)e).get(1)));
+                    print_lineage(((Tuple)e).get(1),false,"    ");
                 }
         case _:
             System.out.println(Printer.print(((Tuple)value).get(0),tp));
-            System.out.println(" <- "+collect_lineage(((Tuple)value).get(1)));
+            print_lineage(((Tuple)value).get(1),false,"    ");
         }
     }
 
diff --git a/core/src/main/java/org/apache/mrql/TopLevel.gen b/core/src/main/java/org/apache/mrql/TopLevel.gen
index b4f1c3a..fe09500 100644
--- a/core/src/main/java/org/apache/mrql/TopLevel.gen
+++ b/core/src/main/java/org/apache/mrql/TopLevel.gen
@@ -124,8 +124,7 @@
             global_vars.remove(v);
         MRData res = expression(e,false);
         global_type_env.insert(v,query_type);
-        if (res instanceof Bag)
-            ((Bag)res).materialize();
+        res.materializeAll();
         new_distributed_binding(v,res);
         return query_plan;
     }
@@ -137,8 +136,7 @@
             global_vars.remove(v);
         MRData res = expression(e,false);
         global_type_env.insert(v,query_type);
-        if (res instanceof Bag)
-            ((Bag)res).materialize();
+        res.materializeAll();
         new_global_binding(v,res);
         return query_plan;
     }
@@ -384,6 +382,7 @@
             long t = System.currentTimeMillis();
             if (expression(e,false) != null && !Config.quiet_execution)
                 System.out.println("Run time: "+(System.currentTimeMillis()-t)/1000.0+" secs");
+            Config.lineage = false;
             if (Config.hadoop_mode)
                 Config.write(Plan.conf);
         case debug(`mode,`e):
diff --git a/core/src/main/java/org/apache/mrql/mrql.lex b/core/src/main/java/org/apache/mrql/mrql.lex
index 39cd33d..f3bb0fd 100644
--- a/core/src/main/java/org/apache/mrql/mrql.lex
+++ b/core/src/main/java/org/apache/mrql/mrql.lex
@@ -61,17 +61,20 @@
 
   public static void start_trace_code () {
     trace_nest++;
-    trace_code += "$(";
+    trace_code += "$("+Tree.line_number+"("+Tree.position_number+") ";
   }
 
   public static String get_trace_code () {
     int loc = trace_code.lastIndexOf("$");
     String s = trace_code.substring(loc+2);
     s = s.substring(0,s.length()-1);
+    int locn = s.indexOf(")");
+    if (loc > 0)
+      locn++;
     trace_nest--;
     if (loc > 0)
       loc--;
-    trace_code = trace_code.substring(0,loc)+s;
+    trace_code = trace_code.substring(0,loc)+s.substring(locn+1);
     return s.replaceAll("\\p{Space}+"," ");
   }