g cannot copy.copy in python3.4+

Author: binux

pyspider/run.py

@@ -223,14 +223,20 @@ def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
               help='Fetcher class to be used.')
 @click.pass_context
 def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
-            timeout, fetcher_cls, async=True, get_object=False, g=None):
+            timeout, fetcher_cls, async=True, get_object=False, no_input=False):
     """
     Run Fetcher.
     """
-    g = g or ctx.obj
+    g = ctx.obj
     Fetcher = load_cls(None, None, fetcher_cls)
 
-    fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor,
+    if no_input:
+        inqueue = None
+        outqueue = None
+    else:
+        inqueue = g.scheduler2fetcher
+        outqueue = g.fetcher2processor
+    fetcher = Fetcher(inqueue=inqueue, outqueue=outqueue,
                       poolsize=poolsize, proxy=proxy, async=async)
     fetcher.phantomjs_proxy = g.phantomjs_proxy
     if user_agent:
@@ -346,10 +352,7 @@ def webui(ctx, host, port, cdn, scheduler_rpc, fetcher_rpc, max_rate, max_burst,
     else:
         # get fetcher instance for webui
         fetcher_config = g.config.get('fetcher', {})
-        mock_g = copy.copy(g)
-        mock_g['scheduler2fetcher'] = None
-        mock_g['fetcher2processor'] = None
-        webui_fetcher = ctx.invoke(fetcher, async=False, get_object=True, g=mock_g, **fetcher_config)
+        webui_fetcher = ctx.invoke(fetcher, async=False, get_object=True, no_input=True, **fetcher_config)
 
         app.config['fetch'] = lambda x: webui_fetcher.fetch(x)