commit 27a299c5c7d195860e63aa0b1d316f3255e4857d
Author: Arturo Filastò <art...@filasto.net>
Date:   Wed Aug 3 13:05:31 2016 +0200

    Add support for initialization of ooniprobe
---
 data/decks/web.yaml           |   1 +
 ooni/agent/scheduler.py       |  25 +++---
 ooni/scripts/ooniprobe.py     |   4 +-
 ooni/settings.py              | 191 ++++++++++++++++++++++++++++++++++++++----
 ooni/tests/__init__.py        |   7 --
 ooni/ui/cli.py                |  52 +++++++++++-
 ooni/ui/web/client/index.html |   2 +-
 ooni/ui/web/server.py         |  82 +++++++++++++++++-
 ooni/utils/__init__.py        |   1 -
 9 files changed, 329 insertions(+), 36 deletions(-)

diff --git a/data/decks/web.yaml b/data/decks/web.yaml
index a81b8f8..c7b9bdc 100644
--- a/data/decks/web.yaml
+++ b/data/decks/web.yaml
@@ -2,6 +2,7 @@
 name: Web related ooniprobe tests
 description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
     Request and the Web Connectivity test
+schedule: "@daily"
 tasks:
 - name: Runs the HTTP Header Field Manipulation test
   ooni:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a6d689f..1f51bd4 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -22,9 +22,11 @@ class ScheduledTask(object):
     schedule = None
     identifier = None
 
-    def __init__(self, schedule=None):
+    def __init__(self, schedule=None, identifier=None):
         if schedule is not None:
             self.schedule = schedule
+        if identifier is not None:
+            self.identifier = identifier
 
         assert self.identifier is not None, "self.identifier must be set"
         assert self.schedule is not None, "self.schedule must be set"
@@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask):
                 measurement_path.child(measurement['id']).remove()
 
 
-class RunDecks(ScheduledTask):
+class RunDeck(ScheduledTask):
     """
     This will run the decks that have been configured on the system as the
     decks to run by default.
     """
-    schedule = '@daily'
-    identifier = 'run-decks'
 
-    def __init__(self, director, schedule=None):
-        super(RunDecks, self).__init__(schedule)
+    def __init__(self, director, deck_id, schedule):
+        self.deck_id = deck_id
         self.director = director
+        identifier = 'run-deck-' + deck_id
+        super(RunDeck, self).__init__(schedule, identifier)
 
     @defer.inlineCallbacks
     def task(self):
-        for deck_id, deck in deck_store.list_enabled():
-            yield deck.setup()
-            yield deck.run(self.director)
+        deck = deck_store.get(self.deck_id)
+        yield deck.setup()
+        yield deck.run(self.director)
 
 class SendHeartBeat(ScheduledTask):
     """
@@ -215,7 +217,10 @@ class SchedulerService(service.MultiService):
         self.schedule(UpdateInputsAndResources())
         self.schedule(UploadReports())
         self.schedule(DeleteOldReports())
-        self.schedule(RunDecks(self.director))
+        for deck_id, deck in deck_store.list_enabled():
+            if deck.schedule is None:
+                continue
+            self.schedule(RunDeck(self.director, deck_id, deck.schedule))
 
         self._looping_call.start(self.interval)
 
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index f5d5b59..430252a 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -6,12 +6,14 @@ from twisted.internet import task, defer
 
 def ooniprobe(reactor):
     from ooni.ui.cli import runWithDaemonDirector, runWithDirector
-    from ooni.ui.cli import setupGlobalOptions
+    from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe
 
     global_options = setupGlobalOptions(logging=True, start_tor=True,
                                         check_incoherences=True)
     if global_options['queue']:
         return runWithDaemonDirector(global_options)
+    elif global_options['initialize']:
+        return initializeOoniprobe(global_options)
     elif global_options['web-ui']:
         from ooni.scripts.ooniprobe_agent import WEB_UI_URL
         from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/settings.py b/ooni/settings.py
index 2161560..8bb3340 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol, 
connectProtocol
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
+
+CONFIG_FILE_TEMPLATE = """\
+# This is the configuration file for OONIProbe
+# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
+# Keep in mind that indentation matters.
+
+basic:
+    # Where OONIProbe should be writing it's log file
+    logfile: {logfile}
+    loglevel: WARNING
+privacy:
+    # Should we include the IP address of the probe in the report?
+    includeip: {include_ip}
+    # Should we include the ASN of the probe in the report?
+    includeasn: {include_asn}
+    # Should we include the country as reported by GeoIP in the report?
+    includecountry: {include_country}
+    # Should we collect a full packet capture on the client?
+    #includepcap: false
+reports:
+    # Should we place a unique ID inside of every report
+    #unique_id: true
+    # This is a prefix for each packet capture file (.pcap) per test:
+    #pcap: null
+    #collector: null
+    # Should we be uploading reports to the collector by default?
+    upload: {should_upload}
+advanced:
+    #debug: false
+    # enable if auto detection fails
+    #tor_binary: /usr/sbin/tor
+    #obfsproxy_binary: /usr/bin/obfsproxy
+    # For auto detection
+    # interface: auto
+    # Of specify a specific interface
+    # interface: wlan0
+    # If you do not specify start_tor, you will have to have Tor running and
+    # explicitly set the control port and SOCKS port
+    #start_tor: true
+    # After how many seconds we should give up on a particular measurement
+    #measurement_timeout: 120
+    # After how many retries we should give up on a measurement
+    #measurement_retries: 2
+    # How many measurements to perform concurrently
+    #measurement_concurrency: 4
+    # After how may seconds we should give up reporting
+    #reporting_timeout: 360
+    # After how many retries to give up on reporting
+    #reporting_retries: 5
+    # How many reports to perform concurrently
+    #reporting_concurrency: 7
+    # If we should support communicating to plaintext backends (via HTTP)
+    # insecure_backend: false
+    # The preferred backend type, can be one of onion, https or cloudfront
+    preferred_backend: {preferred_backend}
+tor:
+    #socks_port: 8801
+    #control_port: 8802
+    # Specify the absolute path to the Tor bridges to use for testing
+    #bridges: bridges.list
+    # Specify path of the tor datadirectory.
+    # This should be set to something to avoid having Tor download each time
+    # the descriptors and consensus data.
+    #data_dir: ~/.tor/
+    #
+    # This is the timeout after which we consider to to not have
+    # bootstrapped properly.
+    #timeout: 200
+    torrc:
+        #HTTPProxy: host:port
+        #HTTPProxyAuthenticator: user:password
+        #HTTPSProxy: host:port
+        #HTTPSProxyAuthenticator: user:password
+        #UseBridges: 1
+        #Bridge:
+        #- "meek_lite 0.0.2.0:1 url=https://meek-reflect.appspot.com/ 
front=www.google.com"
+        #- "meek_lite 0.0.2.0:2 url=https://d2zfqthxsdq309.cloudfront.net/ 
front=a0.awsstatic.com"
+        #- "meek_lite 0.0.2.0:3 url=https://az786092.vo.msecnd.net/ 
front=ajax.aspnetcdn.com"
+        #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy"
+"""
+
+defaults = {
+    "basic": {
+        "loglevel": "WARNING",
+        "logfile": "ooniprobe.log"
+    },
+    "privacy": {
+        "includeip": False,
+        "includeasn": True,
+        "includecountry": True,
+        "includepcap": False
+    },
+    "reports": {
+        "unique_id": True,
+        "pcap": None,
+        "collector": None,
+        "upload": True
+    },
+    "advanced": {
+        "debug": False,
+        "tor_binary": None,
+        "obfsproxy_binary": None,
+        "interface": "auto",
+        "start_tor": True,
+        "measurement_timeout": 120,
+        "measurement_retries": 2,
+        "measurement_concurrency": 4,
+        "reporting_timeout": 360,
+        "reporting_retries": 5,
+        "reporting_concurrency": 7,
+        "insecure_backend": False,
+        "preferred_backend": "onion"
+    },
+    "tor": {
+        "timeout": 200,
+        "torrc": {}
+    }
+}
+
 class OConfig(object):
     _custom_home = None
 
@@ -39,6 +158,17 @@ class OConfig(object):
             return settings.get(category, option)
         return None
 
+    def is_initialized(self):
+        # When this is false it means that the user has not gone
+        # through the steps of acquiring informed consent and
+        # initializing this ooniprobe installation.
+        initialized_path = os.path.join(self.running_path, 'initialized')
+        return os.path.exists(initialized_path)
+
+    def set_initialized(self):
+        initialized_path = os.path.join(self.running_path, 'initialized')
+        with open(initialized_path, 'w+'): pass
+
     @property
     def var_lib_path(self):
         if hasattr(sys, 'real_prefix'):
@@ -149,7 +279,8 @@ class OConfig(object):
             config_file = self.global_options['configfile']
             self.config_file = expanduser(config_file)
         else:
-            self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
+            self.config_file = os.path.join(self.running_path,
+                                            'ooniprobe.conf')
 
         if 'logfile' in self.basic:
             self.basic.logfile = expanduser(
@@ -183,6 +314,33 @@ class OConfig(object):
                 if exc.errno != 17:
                     raise
 
+    def create_config_file(self, include_ip=False, include_asn=True,
+                           include_country=True, should_upload=True,
+                           preferred_backend="onion"):
+        def _bool_to_yaml(value):
+            if value is True:
+                return 'true'
+            elif value is False:
+                return 'false'
+            else:
+                return 'null'
+        # Convert the boolean value to their YAML string representation
+        include_ip = _bool_to_yaml(include_ip )
+        include_asn = _bool_to_yaml(include_asn)
+        include_country = _bool_to_yaml(include_country)
+        should_upload = _bool_to_yaml(should_upload)
+
+        logfile = os.path.join(self.running_path, 'ooniprobe.log')
+        with open(self.config_file, 'w+') as out_file:
+            out_file.write(
+                    CONFIG_FILE_TEMPLATE.format(logfile=logfile,
+                                    include_ip=include_ip,
+                                    include_asn=include_asn,
+                                    include_country=include_country,
+                                    should_upload=should_upload,
+                                    preferred_backend=preferred_backend)
+            )
+        self.read_config_file()
 
     def _create_config_file(self):
         target_config_file = self.config_file
@@ -200,19 +358,24 @@ class OConfig(object):
                         w.write(line)
 
     def read_config_file(self, check_incoherences=False):
-        if not os.path.isfile(self.config_file):
-            print "Configuration file does not exist."
-            self._create_config_file()
-            self.read_config_file()
-
-        with open(self.config_file) as f:
-            config_file_contents = '\n'.join(f.readlines())
-            configuration = yaml.safe_load(config_file_contents)
-
-        for setting in configuration.keys():
-            if setting in dir(self) and configuration[setting] is not None:
-                for k, v in configuration[setting].items():
-                    getattr(self, setting)[k] = v
+        #if not os.path.isfile(self.config_file):
+        #    print "Configuration file does not exist."
+        #    self._create_config_file()
+        #    self.read_config_file()
+
+        configuration = {}
+        if os.path.isfile(self.config_file):
+            with open(self.config_file) as f:
+                config_file_contents = '\n'.join(f.readlines())
+                configuration = yaml.safe_load(config_file_contents)
+
+        for category in defaults.keys():
+            for k, v in defaults[category].items():
+                try:
+                    value = configuration.get(category, {})[k]
+                except KeyError:
+                    value = v
+                getattr(self, category)[k] = value
 
         self.set_paths()
         if check_incoherences:
diff --git a/ooni/tests/__init__.py b/ooni/tests/__init__.py
index e7fd48b..b5dbab4 100644
--- a/ooni/tests/__init__.py
+++ b/ooni/tests/__init__.py
@@ -1,11 +1,4 @@
 import socket
-from ooni.settings import config
-
-config.initialize_ooni_home('ooni_home')
-config.read_config_file()
-config.logging = False
-config.advanced.debug = False
-
 
 def is_internet_connected():
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 3eccf9a..8cd3358 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -30,7 +30,9 @@ class Options(usage.Options):
                 ["list", "s", "List the currently installed ooniprobe "
                               "nettests"],
                 ["verbose", "v", "Show more verbose information"],
-                ["web-ui", "w", "Start the web UI"]
+                ["web-ui", "w", "Start the web UI"],
+                ["initialize", "z", "Initialize ooniprobe to begin running "
+                                    "it"],
                 ]
 
     optParameters = [
@@ -178,10 +180,58 @@ def director_startup_other_failures(failure):
     log.err("An unhandled exception occurred while starting the director!")
     log.exception(failure)
 
+
+def initializeOoniprobe(global_options):
+    # XXX print here the informed consent documentation.
+    answer = raw_input('Should we upload measurements to a collector? (Y/n) ')
+    should_upload = True
+    if answer.lower().startswith("n"):
+        should_upload = False
+
+    answer = raw_input('Should we include your IP in measurements? (y/N) ')
+    include_ip = False
+    if answer.lower().startswith("y"):
+        include_ip = True
+
+    answer = raw_input('Should we include your ASN (your network) in '
+                       'measurements? (Y/n) ')
+    include_asn = False
+    if answer.lower().startswith("n"):
+        include_asn = True
+
+    answer = raw_input('Should we include your Country in '
+                       'measurements? (Y/n) ')
+    include_country = False
+    if answer.lower().startswith("n"):
+        include_country = True
+
+    answer = raw_input('How would you like reports to be uploaded? (onion, '
+                       'https, cloudfronted) ')
+
+    preferred_backend = 'onion'
+    if answer.lower().startswith("https"):
+        preferred_backend = 'https'
+    elif answer.lower().startswith("cloudfronted"):
+        preferred_backend = 'cloudfronted'
+
+    config.create_config_file(include_ip=include_ip,
+                              include_asn=include_asn,
+                              include_country=include_country,
+                              should_upload=should_upload,
+                              preferred_backend=preferred_backend)
+    config.set_initialized()
+
 def setupGlobalOptions(logging, start_tor, check_incoherences):
     global_options = parseOptions()
 
     config.global_options = global_options
+
+    if not config.is_initialized():
+        log.err("You first need to agree to the informed consent and setup "
+                "ooniprobe to run it.")
+        global_options['initialize'] = True
+        return
+
     config.set_paths()
     config.initialize_ooni_home()
     try:
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index e363ba0..6a7c149 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
     <app>
       Loading...
     </app>
-  <script type="text/javascript" 
src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body>
+  <script type="text/javascript" 
src="app.bundle.js?9c4ed560c98eaf61a836"></script></body>
 </html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index a03daa4..ed2193e 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -71,6 +71,39 @@ def xsrf_protect(check=True):
     return deco
 
 
+def _requires_value(value, attrs=[]):
+
+    def deco(f):
+
+        @wraps(f)
+        def wrapper(instance, request, *a, **kw):
+            for attr in attrs:
+                attr_value = getattr(instance, attr)
+                if attr_value is not value:
+                    raise WebUIError(400, "{0} must be {1}".format(attr,
+                                                                   value))
+            return f(instance, request, *a, **kw)
+
+        return wrapper
+
+    return deco
+
+def requires_true(attrs=[]):
+    """
+    This decorator is used to require that a certain set of class attributes 
are
+    set to True.
+    Otherwise it will trigger a WebUIError.
+    """
+    return _requires_value(True, attrs)
+
+def requires_false(attrs=[]):
+    """
+    This decorator is used to require that a certain set of class attributes 
are
+    set to False.
+    Otherwise it will trigger a WebUIError.
+    """
+    return _requires_value(False, attrs)
+
 
 class LongPoller(object):
     def __init__(self, timeout, _reactor=reactor):
@@ -128,6 +161,7 @@ class WebUIAPI(object):
                                     for _ in range(30)])
 
         self._director_started = False
+        self._is_initialized = config.is_initialized()
 
         self.status_poller = LongPoller(
             self._long_polling_timeout, _reactor)
@@ -139,6 +173,10 @@ class WebUIAPI(object):
         self.status_poller.start()
 
         self.director.subscribe(self.handle_director_event)
+        if self._is_initialized:
+            self.start_director()
+
+    def start_director(self):
         d = self.director.start()
 
         d.addCallback(self.director_started)
@@ -151,7 +189,8 @@ class WebUIAPI(object):
             "software_name": "ooniprobe",
             "asn": probe_ip.geodata['asn'],
             "country_code": probe_ip.geodata['countrycode'],
-            "director_started": self._director_started
+            "director_started": self._director_started,
+            "initialized": self._is_initialized
         }
 
     def handle_director_event(self, event):
@@ -208,8 +247,36 @@ class WebUIAPI(object):
         d.addCallback(got_status_update)
         return d
 
+    @app.route('/api/initialize', methods=["POST"])
+    @xsrf_protect(check=True)
+    @requires_false(attrs=['_is_initialized'])
+    def api_initialize(self, request):
+        try:
+            initial_configuration = json.load(request.content)
+        except ValueError:
+            raise WebUIError(400, 'Invalid JSON message recevied')
+
+        required_keys = ['include_ip', 'include_asn', 'include_country',
+                         'should_upload', 'preferred_backend']
+        options = {}
+        for required_key in required_keys:
+            try:
+                options[required_key] = initial_configuration[required_key]
+            except KeyError:
+                raise WebUIError(400, 'Missing required key {0}'.format(
+                    required_key))
+        config.create_config_file(**options)
+        config.set_initialized()
+
+        self._is_initialized = True
+
+        self.status_poller.notify()
+        self.start_director()
+        return self.render_json({"result": "ok"}, request)
+
     @app.route('/api/deck/<string:deck_id>/start', methods=["POST"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_deck_start(self, request, deck_id):
         try:
             deck = self.director.deck_store.get(deck_id)
@@ -225,6 +292,7 @@ class WebUIAPI(object):
 
     @app.route('/api/deck', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_deck_list(self, request):
         deck_list = {
             'available': {},
@@ -246,6 +314,7 @@ class WebUIAPI(object):
 
     @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_deck_enable(self, request, deck_id):
         try:
             self.director.deck_store.enable(deck_id)
@@ -256,6 +325,7 @@ class WebUIAPI(object):
 
     @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_deck_disable(self, request, deck_id):
         try:
             self.director.deck_store.disable(deck_id)
@@ -276,6 +346,7 @@ class WebUIAPI(object):
 
     @app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_nettest_start(self, request, test_name):
         try:
             _ = self.director.netTests[test_name]
@@ -321,11 +392,13 @@ class WebUIAPI(object):
 
     @app.route('/api/nettest', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_nettest_list(self, request):
         return self.render_json(self.director.netTests, request)
 
     @app.route('/api/input', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_input_list(self, request):
         input_store_list = self.director.input_store.list()
         for key, value in input_store_list.items():
@@ -334,6 +407,7 @@ class WebUIAPI(object):
 
     @app.route('/api/input/<string:input_id>/content', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_input_content(self, request, input_id):
         content = self.director.input_store.getContent(input_id)
         request.setHeader('Content-Type', 'text/plain')
@@ -342,6 +416,7 @@ class WebUIAPI(object):
 
     @app.route('/api/input/<string:input_id>', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_input_details(self, request, input_id):
         return self.render_json(
             self.director.input_store.get(input_id), request
@@ -349,12 +424,14 @@ class WebUIAPI(object):
 
     @app.route('/api/measurement', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_measurement_list(self, request):
         measurements = list_measurements()
         return self.render_json({"measurements": measurements}, request)
 
     @app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_measurement_summary(self, request, measurement_id):
         try:
             measurement = get_measurement(measurement_id)
@@ -373,6 +450,7 @@ class WebUIAPI(object):
 
     @app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_is_initialized'])
     def api_measurement_delete(self, request, measurement_id):
         try:
             measurement = get_measurement(measurement_id)
@@ -394,6 +472,7 @@ class WebUIAPI(object):
 
     @app.route('/api/measurement/<string:measurement_id>/keep', 
methods=["POST"])
     @xsrf_protect(check=True)
+    @requires_true(attrs=['_is_initialized'])
     def api_measurement_keep(self, request, measurement_id):
         try:
             measurement_dir = self.measurement_path.child(measurement_id)
@@ -409,6 +488,7 @@ class WebUIAPI(object):
     @app.route('/api/measurement/<string:measurement_id>/<int:idx>',
                methods=["GET"])
     @xsrf_protect(check=False)
+    @requires_true(attrs=['_is_initialized'])
     def api_measurement_view(self, request, measurement_id, idx):
         try:
             measurement_dir = self.measurement_path.child(measurement_id)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 35d419d..d672ca8 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -56,7 +56,6 @@ class Storage(dict):
         for (k, v) in value.items():
             self[k] = v
 
-
 def checkForRoot():
     if os.getuid() != 0:
         raise errors.InsufficientPrivileges



_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Reply via email to