Properly handle -q and --debug flags. Added a test for verifying -q.

sf.py

@@ -58,6 +58,10 @@
 error_handler.setFormatter(log_format)
 log.addHandler(error_handler)
 
+console_handler = logging.StreamHandler(sys.stderr)
+console_handler.setFormatter(log_format)
+log.addHandler(console_handler)
+
 scanId = None
 dbh = None
 
@@ -146,10 +150,6 @@ def main():
     if args.q or args.o == "json":
         log.setLevel(logging.NOTSET)
         sfConfig['__logging'] = False
-    else:
-        console_handler = logging.StreamHandler(sys.stderr)
-        console_handler.setFormatter(log_format)
-        log.addHandler(console_handler)
 
     sfModules = dict()
     sft = SpiderFoot(sfConfig)

spiderfoot/plugin.py

@@ -2,7 +2,6 @@
 import threading
 import queue
 from time import sleep
-from copy import copy
 
 
 class SpiderFootPlugin():
@@ -55,6 +54,10 @@ class SpiderFootPlugin():
     incomingEventQueue = None
     # Queue for produced events
     outgoingEventQueue = None
+    # SpiderFoot object, set in each module's setup() function
+    sf = None
+    # Configuration, set in each module's setup() function
+    opts = dict()
 
     def __init__(self):
         """Not really needed in most cases."""
@@ -276,7 +279,7 @@ def notifyListeners(self, sfEvent):
                 try:
                     listener.handleEvent(sfEvent)
                 except Exception as e:
-                    self.log.exception(f"Module ({listener.__module__}) encountered an error: {e}")
+                    self.sf.error(f"Module ({listener.__module__}) encountered an error: {e}")
 
     def checkForStop(self):
         """For modules to use to check for when they should give back control.
@@ -343,30 +346,29 @@ def threadWorker(self):
             # create new database handle since we're in our own thread
             from spiderfoot import SpiderFootDb
             self.setDbh(SpiderFootDb(self.opts))
-            self.sf = copy(self.sf)
             self.sf._dbh = self.__sfdb__
 
             if not (self.incomingEventQueue and self.outgoingEventQueue):
-                self.log.error("Please set up queues before starting module as thread")
+                self.sf.error("Please set up queues before starting module as thread")
                 return
 
             while not self.checkForStop():
                 try:
                     sfEvent = self.incomingEventQueue.get_nowait()
-                    self.log.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")
+                    self.sf.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")
                     self.running = True
                     self.handleEvent(sfEvent)
                     self.running = False
                 except queue.Empty:
                     sleep(.3)
                     continue
         except KeyboardInterrupt:
-            self.log.warning(f"Interrupted module {self.__name__}.")
+            self.sf.debug(f"Interrupted module {self.__name__}.")
             self._stopScanning = True
         except Exception as e:
             import traceback
-            self.log.error(f"Exception ({e.__class__.__name__}) in module {self.__name__}."
-                           + traceback.format_exc())
+            self.sf.error(f"Exception ({e.__class__.__name__}) in module {self.__name__}."
+                          + traceback.format_exc())
             self.errorState = True
         finally:
             self.running = False

test/integration/test_sf.py

@@ -79,6 +79,11 @@ def test_debug_arg_should_enable_and_print_debug_output(self):
         self.assertIn(b"modules.sfp__stor_db : Storing an event: ROOT", err)
         self.assertEqual(0, code)
 
+    def test_quiet_arg_should_hide_debug_output(self):
+        out, err, code = self.execute([sys.executable, "sf.py", "-q", "-m", "example module", "-s", "spiderfoot.net"])
+        self.assertNotIn(b"[INFO]", err)
+        self.assertEqual(0, code)
+
     def test_run_scan_invalid_target_should_exit(self):
         invalid_target = '.'
         out, err, code = self.execute([sys.executable, "sf.py", "-s", invalid_target])