commit 27a299c5c7d195860e63aa0b1d316f3255e4857d
Author: Arturo Filastò <art...@filasto.net>
Date: Wed Aug 3 13:05:31 2016 +0200
Add support for initialization of ooniprobe
---
data/decks/web.yaml | 1 +
ooni/agent/scheduler.py | 25 +++---
ooni/scripts/ooniprobe.py | 4 +-
ooni/settings.py | 191 ++++++++++++++++++++++++++++++++++++++----
ooni/tests/__init__.py | 7 --
ooni/ui/cli.py | 52 +++++++++++-
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 82 +++++++++++++++++-
ooni/utils/__init__.py | 1 -
9 files changed, 329 insertions(+), 36 deletions(-)
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
index a81b8f8..c7b9bdc 100644
--- a/data/decks/web.yaml
+++ b/data/decks/web.yaml
@@ -2,6 +2,7 @@
name: Web related ooniprobe tests
description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
Request and the Web Connectivity test
+schedule: "@daily"
tasks:
- name: Runs the HTTP Header Field Manipulation test
ooni:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a6d689f..1f51bd4 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -22,9 +22,11 @@ class ScheduledTask(object):
schedule = None
identifier = None
- def __init__(self, schedule=None):
+ def __init__(self, schedule=None, identifier=None):
if schedule is not None:
self.schedule = schedule
+ if identifier is not None:
+ self.identifier = identifier
assert self.identifier is not None, "self.identifier must be set"
assert self.schedule is not None, "self.schedule must be set"
@@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask):
measurement_path.child(measurement['id']).remove()
-class RunDecks(ScheduledTask):
+class RunDeck(ScheduledTask):
"""
This will run the decks that have been configured on the system as the
decks to run by default.
"""
- schedule = '@daily'
- identifier = 'run-decks'
- def __init__(self, director, schedule=None):
- super(RunDecks, self).__init__(schedule)
+ def __init__(self, director, deck_id, schedule):
+ self.deck_id = deck_id
self.director = director
+ identifier = 'run-deck-' + deck_id
+ super(RunDeck, self).__init__(schedule, identifier)
@defer.inlineCallbacks
def task(self):
- for deck_id, deck in deck_store.list_enabled():
- yield deck.setup()
- yield deck.run(self.director)
+ deck = deck_store.get(self.deck_id)
+ yield deck.setup()
+ yield deck.run(self.director)
class SendHeartBeat(ScheduledTask):
"""
@@ -215,7 +217,10 @@ class SchedulerService(service.MultiService):
self.schedule(UpdateInputsAndResources())
self.schedule(UploadReports())
self.schedule(DeleteOldReports())
- self.schedule(RunDecks(self.director))
+ for deck_id, deck in deck_store.list_enabled():
+ if deck.schedule is None:
+ continue
+ self.schedule(RunDeck(self.director, deck_id, deck.schedule))
self._looping_call.start(self.interval)
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index f5d5b59..430252a 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -6,12 +6,14 @@ from twisted.internet import task, defer
def ooniprobe(reactor):
from ooni.ui.cli import runWithDaemonDirector, runWithDirector
- from ooni.ui.cli import setupGlobalOptions
+ from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe
global_options = setupGlobalOptions(logging=True, start_tor=True,
check_incoherences=True)
if global_options['queue']:
return runWithDaemonDirector(global_options)
+ elif global_options['initialize']:
+ return initializeOoniprobe(global_options)
elif global_options['web-ui']:
from ooni.scripts.ooniprobe_agent import WEB_UI_URL
from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/settings.py b/ooni/settings.py
index 2161560..8bb3340 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol,
connectProtocol
from ooni.utils import Storage, log, get_ooni_root
from ooni import errors
+
+CONFIG_FILE_TEMPLATE = """\
+# This is the configuration file for OONIProbe
+# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
+# Keep in mind that indentation matters.
+
+basic:
+ # Where OONIProbe should be writing it's log file
+ logfile: {logfile}
+ loglevel: WARNING
+privacy:
+ # Should we include the IP address of the probe in the report?
+ includeip: {include_ip}
+ # Should we include the ASN of the probe in the report?
+ includeasn: {include_asn}
+ # Should we include the country as reported by GeoIP in the report?
+ includecountry: {include_country}
+ # Should we collect a full packet capture on the client?
+ #includepcap: false
+reports:
+ # Should we place a unique ID inside of every report
+ #unique_id: true
+ # This is a prefix for each packet capture file (.pcap) per test:
+ #pcap: null
+ #collector: null
+ # Should we be uploading reports to the collector by default?
+ upload: {should_upload}
+advanced:
+ #debug: false
+ # enable if auto detection fails
+ #tor_binary: /usr/sbin/tor
+ #obfsproxy_binary: /usr/bin/obfsproxy
+ # For auto detection
+ # interface: auto
+ # Of specify a specific interface
+ # interface: wlan0
+ # If you do not specify start_tor, you will have to have Tor running and
+ # explicitly set the control port and SOCKS port
+ #start_tor: true
+ # After how many seconds we should give up on a particular measurement
+ #measurement_timeout: 120
+ # After how many retries we should give up on a measurement
+ #measurement_retries: 2
+ # How many measurements to perform concurrently
+ #measurement_concurrency: 4
+ # After how may seconds we should give up reporting
+ #reporting_timeout: 360
+ # After how many retries to give up on reporting
+ #reporting_retries: 5
+ # How many reports to perform concurrently
+ #reporting_concurrency: 7
+ # If we should support communicating to plaintext backends (via HTTP)
+ # insecure_backend: false
+ # The preferred backend type, can be one of onion, https or cloudfront
+ preferred_backend: {preferred_backend}
+tor:
+ #socks_port: 8801
+ #control_port: 8802
+ # Specify the absolute path to the Tor bridges to use for testing
+ #bridges: bridges.list
+ # Specify path of the tor datadirectory.
+ # This should be set to something to avoid having Tor download each time
+ # the descriptors and consensus data.
+ #data_dir: ~/.tor/
+ #
+ # This is the timeout after which we consider to to not have
+ # bootstrapped properly.
+ #timeout: 200
+ torrc:
+ #HTTPProxy: host:port
+ #HTTPProxyAuthenticator: user:password
+ #HTTPSProxy: host:port
+ #HTTPSProxyAuthenticator: user:password
+ #UseBridges: 1
+ #Bridge:
+ #- "meek_lite 0.0.2.0:1 url=https://meek-reflect.appspot.com/
front=www.google.com"
+ #- "meek_lite 0.0.2.0:2 url=https://d2zfqthxsdq309.cloudfront.net/
front=a0.awsstatic.com"
+ #- "meek_lite 0.0.2.0:3 url=https://az786092.vo.msecnd.net/
front=ajax.aspnetcdn.com"
+ #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy"
+"""
+
+defaults = {
+ "basic": {
+ "loglevel": "WARNING",
+ "logfile": "ooniprobe.log"
+ },
+ "privacy": {
+ "includeip": False,
+ "includeasn": True,
+ "includecountry": True,
+ "includepcap": False
+ },
+ "reports": {
+ "unique_id": True,
+ "pcap": None,
+ "collector": None,
+ "upload": True
+ },
+ "advanced": {
+ "debug": False,
+ "tor_binary": None,
+ "obfsproxy_binary": None,
+ "interface": "auto",
+ "start_tor": True,
+ "measurement_timeout": 120,
+ "measurement_retries": 2,
+ "measurement_concurrency": 4,
+ "reporting_timeout": 360,
+ "reporting_retries": 5,
+ "reporting_concurrency": 7,
+ "insecure_backend": False,
+ "preferred_backend": "onion"
+ },
+ "tor": {
+ "timeout": 200,
+ "torrc": {}
+ }
+}
+
class OConfig(object):
_custom_home = None
@@ -39,6 +158,17 @@ class OConfig(object):
return settings.get(category, option)
return None
+ def is_initialized(self):
+ # When this is false it means that the user has not gone
+ # through the steps of acquiring informed consent and
+ # initializing this ooniprobe installation.
+ initialized_path = os.path.join(self.running_path, 'initialized')
+ return os.path.exists(initialized_path)
+
+ def set_initialized(self):
+ initialized_path = os.path.join(self.running_path, 'initialized')
+ with open(initialized_path, 'w+'): pass
+
@property
def var_lib_path(self):
if hasattr(sys, 'real_prefix'):
@@ -149,7 +279,8 @@ class OConfig(object):
config_file = self.global_options['configfile']
self.config_file = expanduser(config_file)
else:
- self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
+ self.config_file = os.path.join(self.running_path,
+ 'ooniprobe.conf')
if 'logfile' in self.basic:
self.basic.logfile = expanduser(
@@ -183,6 +314,33 @@ class OConfig(object):
if exc.errno != 17:
raise
+ def create_config_file(self, include_ip=False, include_asn=True,
+ include_country=True, should_upload=True,
+ preferred_backend="onion"):
+ def _bool_to_yaml(value):
+ if value is True:
+ return 'true'
+ elif value is False:
+ return 'false'
+ else:
+ return 'null'
+ # Convert the boolean value to their YAML string representation
+ include_ip = _bool_to_yaml(include_ip )
+ include_asn = _bool_to_yaml(include_asn)
+ include_country = _bool_to_yaml(include_country)
+ should_upload = _bool_to_yaml(should_upload)
+
+ logfile = os.path.join(self.running_path, 'ooniprobe.log')
+ with open(self.config_file, 'w+') as out_file:
+ out_file.write(
+ CONFIG_FILE_TEMPLATE.format(logfile=logfile,
+ include_ip=include_ip,
+ include_asn=include_asn,
+ include_country=include_country,
+ should_upload=should_upload,
+ preferred_backend=preferred_backend)
+ )
+ self.read_config_file()
def _create_config_file(self):
target_config_file = self.config_file
@@ -200,19 +358,24 @@ class OConfig(object):
w.write(line)
def read_config_file(self, check_incoherences=False):
- if not os.path.isfile(self.config_file):
- print "Configuration file does not exist."
- self._create_config_file()
- self.read_config_file()
-
- with open(self.config_file) as f:
- config_file_contents = '\n'.join(f.readlines())
- configuration = yaml.safe_load(config_file_contents)
-
- for setting in configuration.keys():
- if setting in dir(self) and configuration[setting] is not None:
- for k, v in configuration[setting].items():
- getattr(self, setting)[k] = v
+ #if not os.path.isfile(self.config_file):
+ # print "Configuration file does not exist."
+ # self._create_config_file()
+ # self.read_config_file()
+
+ configuration = {}
+ if os.path.isfile(self.config_file):
+ with open(self.config_file) as f:
+ config_file_contents = '\n'.join(f.readlines())
+ configuration = yaml.safe_load(config_file_contents)
+
+ for category in defaults.keys():
+ for k, v in defaults[category].items():
+ try:
+ value = configuration.get(category, {})[k]
+ except KeyError:
+ value = v
+ getattr(self, category)[k] = value
self.set_paths()
if check_incoherences:
diff --git a/ooni/tests/__init__.py b/ooni/tests/__init__.py
index e7fd48b..b5dbab4 100644
--- a/ooni/tests/__init__.py
+++ b/ooni/tests/__init__.py
@@ -1,11 +1,4 @@
import socket
-from ooni.settings import config
-
-config.initialize_ooni_home('ooni_home')
-config.read_config_file()
-config.logging = False
-config.advanced.debug = False
-
def is_internet_connected():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 3eccf9a..8cd3358 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -30,7 +30,9 @@ class Options(usage.Options):
["list", "s", "List the currently installed ooniprobe "
"nettests"],
["verbose", "v", "Show more verbose information"],
- ["web-ui", "w", "Start the web UI"]
+ ["web-ui", "w", "Start the web UI"],
+ ["initialize", "z", "Initialize ooniprobe to begin running "
+ "it"],
]
optParameters = [
@@ -178,10 +180,58 @@ def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
+
+def initializeOoniprobe(global_options):
+ # XXX print here the informed consent documentation.
+ answer = raw_input('Should we upload measurements to a collector? (Y/n) ')
+ should_upload = True
+ if answer.lower().startswith("n"):
+ should_upload = False
+
+ answer = raw_input('Should we include your IP in measurements? (y/N) ')
+ include_ip = False
+ if answer.lower().startswith("y"):
+ include_ip = True
+
+ answer = raw_input('Should we include your ASN (your network) in '
+ 'measurements? (Y/n) ')
+ include_asn = False
+ if answer.lower().startswith("n"):
+ include_asn = True
+
+ answer = raw_input('Should we include your Country in '
+ 'measurements? (Y/n) ')
+ include_country = False
+ if answer.lower().startswith("n"):
+ include_country = True
+
+ answer = raw_input('How would you like reports to be uploaded? (onion, '
+ 'https, cloudfronted) ')
+
+ preferred_backend = 'onion'
+ if answer.lower().startswith("https"):
+ preferred_backend = 'https'
+ elif answer.lower().startswith("cloudfronted"):
+ preferred_backend = 'cloudfronted'
+
+ config.create_config_file(include_ip=include_ip,
+ include_asn=include_asn,
+ include_country=include_country,
+ should_upload=should_upload,
+ preferred_backend=preferred_backend)
+ config.set_initialized()
+
def setupGlobalOptions(logging, start_tor, check_incoherences):
global_options = parseOptions()
config.global_options = global_options
+
+ if not config.is_initialized():
+ log.err("You first need to agree to the informed consent and setup "
+ "ooniprobe to run it.")
+ global_options['initialize'] = True
+ return
+
config.set_paths()
config.initialize_ooni_home()
try:
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index e363ba0..6a7c149 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript"
src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body>
+ <script type="text/javascript"
src="app.bundle.js?9c4ed560c98eaf61a836"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index a03daa4..ed2193e 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -71,6 +71,39 @@ def xsrf_protect(check=True):
return deco
+def _requires_value(value, attrs=[]):
+
+ def deco(f):
+
+ @wraps(f)
+ def wrapper(instance, request, *a, **kw):
+ for attr in attrs:
+ attr_value = getattr(instance, attr)
+ if attr_value is not value:
+ raise WebUIError(400, "{0} must be {1}".format(attr,
+ value))
+ return f(instance, request, *a, **kw)
+
+ return wrapper
+
+ return deco
+
+def requires_true(attrs=[]):
+ """
+ This decorator is used to require that a certain set of class attributes
are
+ set to True.
+ Otherwise it will trigger a WebUIError.
+ """
+ return _requires_value(True, attrs)
+
+def requires_false(attrs=[]):
+ """
+ This decorator is used to require that a certain set of class attributes
are
+ set to False.
+ Otherwise it will trigger a WebUIError.
+ """
+ return _requires_value(False, attrs)
+
class LongPoller(object):
def __init__(self, timeout, _reactor=reactor):
@@ -128,6 +161,7 @@ class WebUIAPI(object):
for _ in range(30)])
self._director_started = False
+ self._is_initialized = config.is_initialized()
self.status_poller = LongPoller(
self._long_polling_timeout, _reactor)
@@ -139,6 +173,10 @@ class WebUIAPI(object):
self.status_poller.start()
self.director.subscribe(self.handle_director_event)
+ if self._is_initialized:
+ self.start_director()
+
+ def start_director(self):
d = self.director.start()
d.addCallback(self.director_started)
@@ -151,7 +189,8 @@ class WebUIAPI(object):
"software_name": "ooniprobe",
"asn": probe_ip.geodata['asn'],
"country_code": probe_ip.geodata['countrycode'],
- "director_started": self._director_started
+ "director_started": self._director_started,
+ "initialized": self._is_initialized
}
def handle_director_event(self, event):
@@ -208,8 +247,36 @@ class WebUIAPI(object):
d.addCallback(got_status_update)
return d
+ @app.route('/api/initialize', methods=["POST"])
+ @xsrf_protect(check=True)
+ @requires_false(attrs=['_is_initialized'])
+ def api_initialize(self, request):
+ try:
+ initial_configuration = json.load(request.content)
+ except ValueError:
+ raise WebUIError(400, 'Invalid JSON message recevied')
+
+ required_keys = ['include_ip', 'include_asn', 'include_country',
+ 'should_upload', 'preferred_backend']
+ options = {}
+ for required_key in required_keys:
+ try:
+ options[required_key] = initial_configuration[required_key]
+ except KeyError:
+ raise WebUIError(400, 'Missing required key {0}'.format(
+ required_key))
+ config.create_config_file(**options)
+ config.set_initialized()
+
+ self._is_initialized = True
+
+ self.status_poller.notify()
+ self.start_director()
+ return self.render_json({"result": "ok"}, request)
+
@app.route('/api/deck/<string:deck_id>/start', methods=["POST"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_start(self, request, deck_id):
try:
deck = self.director.deck_store.get(deck_id)
@@ -225,6 +292,7 @@ class WebUIAPI(object):
@app.route('/api/deck', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_list(self, request):
deck_list = {
'available': {},
@@ -246,6 +314,7 @@ class WebUIAPI(object):
@app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_enable(self, request, deck_id):
try:
self.director.deck_store.enable(deck_id)
@@ -256,6 +325,7 @@ class WebUIAPI(object):
@app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_disable(self, request, deck_id):
try:
self.director.deck_store.disable(deck_id)
@@ -276,6 +346,7 @@ class WebUIAPI(object):
@app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_nettest_start(self, request, test_name):
try:
_ = self.director.netTests[test_name]
@@ -321,11 +392,13 @@ class WebUIAPI(object):
@app.route('/api/nettest', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_director_started', '_is_initialized'])
def api_nettest_list(self, request):
return self.render_json(self.director.netTests, request)
@app.route('/api/input', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_input_list(self, request):
input_store_list = self.director.input_store.list()
for key, value in input_store_list.items():
@@ -334,6 +407,7 @@ class WebUIAPI(object):
@app.route('/api/input/<string:input_id>/content', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_input_content(self, request, input_id):
content = self.director.input_store.getContent(input_id)
request.setHeader('Content-Type', 'text/plain')
@@ -342,6 +416,7 @@ class WebUIAPI(object):
@app.route('/api/input/<string:input_id>', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_input_details(self, request, input_id):
return self.render_json(
self.director.input_store.get(input_id), request
@@ -349,12 +424,14 @@ class WebUIAPI(object):
@app.route('/api/measurement', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_measurement_list(self, request):
measurements = list_measurements()
return self.render_json({"measurements": measurements}, request)
@app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_measurement_summary(self, request, measurement_id):
try:
measurement = get_measurement(measurement_id)
@@ -373,6 +450,7 @@ class WebUIAPI(object):
@app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_is_initialized'])
def api_measurement_delete(self, request, measurement_id):
try:
measurement = get_measurement(measurement_id)
@@ -394,6 +472,7 @@ class WebUIAPI(object):
@app.route('/api/measurement/<string:measurement_id>/keep',
methods=["POST"])
@xsrf_protect(check=True)
+ @requires_true(attrs=['_is_initialized'])
def api_measurement_keep(self, request, measurement_id):
try:
measurement_dir = self.measurement_path.child(measurement_id)
@@ -409,6 +488,7 @@ class WebUIAPI(object):
@app.route('/api/measurement/<string:measurement_id>/<int:idx>',
methods=["GET"])
@xsrf_protect(check=False)
+ @requires_true(attrs=['_is_initialized'])
def api_measurement_view(self, request, measurement_id, idx):
try:
measurement_dir = self.measurement_path.child(measurement_id)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 35d419d..d672ca8 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -56,7 +56,6 @@ class Storage(dict):
for (k, v) in value.items():
self[k] = v
-
def checkForRoot():
if os.getuid() != 0:
raise errors.InsufficientPrivileges
_______________________________________________
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
