[tor-commits] [stem/master] Stacktrace when queue module is present

[atagar]

commit 03f32518527070f4584aa52b7ce2ea11028cc2ba
Author: Damian Johnson 
Date:   Mon Sep 19 22:12:30 2016 -0700

Stacktrace when queue module is present

When running with the python 2.x if you have the queue module we use
io.StringIO rather than StringIO.StringIO. This causes two of our 
Controller's
methods to fail with...

  Traceback (most recent call last):
File "circuits.py", line 7, in 
  for circ in sorted(controller.get_circuits()):
File "/usr/lib/python2.7/site-packages/stem/control.py", line 455, in 
wrapped
  return func(self, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/stem/control.py", line 3284, in 
get_circuits
  circ_message = stem.socket.recv_message(StringIO('650 CIRC ' + circ + 
'\r\n'))
  TypeError: initial_value must be unicode or None, not str

Thanks to mancha for catching this!
---
 docs/change_log.rst   | 4 ++--
 stem/control.py   | 7 +++
 stem/response/__init__.py | 8 ++--
 3 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst
index 7ae3466..2959f6b 100644
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -3,7 +3,7 @@ Change Log
 
 The following is a log of all user-facing changes to Stem, both released and
 unreleased. For a monthly report on work being done see my `development log
-`_.
+`_.
 
 * :ref:`versioning`
 * :ref:`unreleased`
@@ -107,7 +107,7 @@ The following are only available within Stem's `git 
repository
 
   * `Comparison of our descriptor parsing libraries 
`_
   * Example for `custom path selection for circuits 
`_ (:trac:`8728`)
-  * Example for `persisting ephemeral hidden service keys 
<`_
+  * Example for `persisting ephemeral hidden service keys 
`_
 
 .. _version_1.4:
 
diff --git a/stem/control.py b/stem/control.py
index dfa5add..af22c8f 100644
--- a/stem/control.py
+++ b/stem/control.py
@@ -252,11 +252,10 @@ except ImportError:
   from stem.util.ordereddict import OrderedDict
 
 try:
+  # Added in 3.x
   import queue
-  from io import StringIO
 except ImportError:
   import Queue as queue
-  from StringIO import StringIO
 
 import stem.descriptor.microdescriptor
 import stem.descriptor.reader
@@ -3281,7 +3280,7 @@ class Controller(BaseController):
 response = self.get_info('circuit-status')
 
 for circ in response.splitlines():
-  circ_message = stem.socket.recv_message(StringIO('650 CIRC ' + circ + 
'\r\n'))
+  circ_message = 
stem.socket.recv_message(io.StringIO(stem.util.str_tools._to_unicode('650 CIRC 
' + circ + '\r\n')))
   stem.response.convert('EVENT', circ_message, arrived_at = 0)
   circuits.append(circ_message)
 
@@ -3464,7 +3463,7 @@ class Controller(BaseController):
 response = self.get_info('stream-status')
 
 for stream in response.splitlines():
-  message = stem.socket.recv_message(StringIO('650 STREAM ' + stream + 
'\r\n'))
+  message = 
stem.socket.recv_message(io.StringIO(stem.util.str_tools._to_unicode('650 
STREAM ' + stream + '\r\n')))
   stem.response.convert('EVENT', message, arrived_at = 0)
   streams.append(message)
 
diff --git a/stem/response/__init__.py b/stem/response/__init__.py
index b6715fc..3300d41 100644
--- a/stem/response/__init__.py
+++ b/stem/response/__init__.py
@@ -30,6 +30,7 @@ Parses replies from the control socket.
 +- pop_mapping - removes and returns the next entry as a KEY=VALUE mapping
 """
 
+import io
 import re
 import threading
 
@@ -48,11 +49,6 @@ __all__ = [
   'SingleLineResponse',
 ]
 
-try:
-  from StringIO import StringIO
-except ImportError:
-  from io import StringIO
-
 KEY_ARG = re.compile('^(\S+)=')
 
 # Escape sequences from the 'esc_for_log' function of tor's 'common/util.c'.
@@ -157,7 +153,7 @@ class ControlMessage(object):
 :returns: stem.response.ControlMessage instance
 """
 
-msg = stem.socket.recv_message(StringIO(content))
+msg = 
stem.socket.recv_message(io.StringIO(stem.util.str_tools._to_unicode(content)))
 
 if msg_type is not None:
   convert(msg_type, msg, **kwargs)

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] Add support for AES256 and AES192

[nickm]

commit 6cb9c2cf77cc8375f89cc8d625d0b60e292d8160
Author: Nick Mathewson 
Date:   Fri Sep 16 11:21:33 2016 -0400

Add support for AES256 and AES192

(This will be used by prop224)
---
 src/common/aes.c   |  2 +-
 src/common/crypto.c| 39 +++
 src/common/crypto.h|  4 
 src/test/test_crypto.c | 62 ++
 4 files changed, 82 insertions(+), 25 deletions(-)

diff --git a/src/common/aes.c b/src/common/aes.c
index 7131ce1..cb8fb68 100644
--- a/src/common/aes.c
+++ b/src/common/aes.c
@@ -48,7 +48,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 
 /* We have five strategies for implementing AES counter mode.
  *
- * Best with x86 and x86_64: Use EVP_aes_ctr128() and EVP_EncryptUpdate().
+ * Best with x86 and x86_64: Use EVP_aes_*_ctr() and EVP_EncryptUpdate().
  * This is possible with OpenSSL 1.0.1, where the counter-mode implementation
  * can use bit-sliced or vectorized AES or AESNI as appropriate.
  *
diff --git a/src/common/crypto.c b/src/common/crypto.c
index fb77340..72c1c45 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -542,29 +542,48 @@ crypto_pk_free(crypto_pk_t *env)
 }
 
 /** Allocate and return a new symmetric cipher using the provided key and iv.
- * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  Both
- * must be provided.
- */
+ * The key is bits bits long; the IV is CIPHER_IV_LEN bytes.  Both
+ * must be provided. Key length must be 128, 192, or 256 */
 crypto_cipher_t *
-crypto_cipher_new_with_iv(const char *key, const char *iv)
+crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
+   const uint8_t *iv,
+   int bits)
 {
-  crypto_cipher_t *env;
   tor_assert(key);
   tor_assert(iv);
 
-  env = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128);
+  return aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, bits);
+}
 
-  return env;
+/** Allocate and return a new symmetric cipher using the provided key and iv.
+ * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  Both
+ * must be provided.
+ */
+crypto_cipher_t *
+crypto_cipher_new_with_iv(const char *key, const char *iv)
+{
+  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)iv,
+128);
 }
 
 /** Return a new crypto_cipher_t with the provided key and an IV of all
- * zero bytes.  */
+ * zero bytes and key length bits.  Key length must be 128, 192, or
+ * 256. */
 crypto_cipher_t *
-crypto_cipher_new(const char *key)
+crypto_cipher_new_with_bits(const char *key, int bits)
 {
   char zeroiv[CIPHER_IV_LEN];
   memset(zeroiv, 0, sizeof(zeroiv));
-  return crypto_cipher_new_with_iv(key, zeroiv);
+  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)zeroiv,
+bits);
+}
+
+/** Return a new crypto_cipher_t with the provided key (of
+ * CIPHER_KEY_LEN bytes) and an IV of all zero bytes.  */
+crypto_cipher_t *
+crypto_cipher_new(const char *key)
+{
+  return crypto_cipher_new_with_bits(key, 128);
 }
 
 /** Free a symmetric cipher.
diff --git a/src/common/crypto.h b/src/common/crypto.h
index e60cf34..116e0a6 100644
--- a/src/common/crypto.h
+++ b/src/common/crypto.h
@@ -138,7 +138,11 @@ void crypto_pk_free(crypto_pk_t *env);
 
 void crypto_set_tls_dh_prime(void);
 crypto_cipher_t *crypto_cipher_new(const char *key);
+crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits);
 crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv);
+crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
+const uint8_t *iv,
+int bits);
 void crypto_cipher_free(crypto_cipher_t *env);
 
 /* public key crypto */
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 6586c06..64a46f7 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -363,9 +363,9 @@ test_crypto_rng_engine(void *arg)
   ;
 }
 
-/** Run unit tests for our AES functionality */
+/** Run unit tests for our AES128 functionality */
 static void
-test_crypto_aes(void *arg)
+test_crypto_aes128(void *arg)
 {
   char *data1 = NULL, *data2 = NULL, *data3 = NULL;
   crypto_cipher_t *env1 = NULL, *env2 = NULL;
@@ -514,32 +514,61 @@ test_crypto_aes(void *arg)
 static void
 test_crypto_aes_ctr_testvec(void *arg)
 {
-  (void)arg;
+  const char *bitstr = arg;
   char *mem_op_hex_tmp=NULL;
+  crypto_cipher_t *c=NULL;
 
   /* from NIST SP800-38a, section F.5 */
-  const char key16[] = "2b7e151628aed2a6abf7158809cf4f3c";
   const char ctr16[] = "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
   const char plaintext16[] =
 "6bc1bee22e409f96e93d7e117393172a"
 "ae2d8a571e03ac9c9eb76fac45af8e51"
 "30c81c46a35ce411e5fbc1191a0a52ef"
 "f69f2445df4f9b17ad2b417be66c3710";
-  const char ciphertext16[] 

[tor-commits] [tor/master] In aes.c, support 192-bit and 256-bit keys.

[nickm]

commit 981d0a24b81f27a642946648e49b3cadbd0c28b7
Author: Nick Mathewson 
Date:   Fri Sep 16 09:51:51 2016 -0400

In aes.c, support 192-bit and 256-bit keys.

Also, change the input types for aes_new_cipher to be unsigned,
as they should have been all along.
---
 src/common/aes.c| 26 +-
 src/common/aes.h|  3 ++-
 src/common/crypto.c | 13 +++--
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/common/aes.c b/src/common/aes.c
index 2b8a68c..7131ce1 100644
--- a/src/common/aes.c
+++ b/src/common/aes.c
@@ -89,11 +89,17 @@ ENABLE_GCC_WARNING(redundant-decls)
 /* We don't actually define the struct here. */
 
 aes_cnt_cipher_t *
-aes_new_cipher(const char *key, const char *iv)
+aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits)
 {
   EVP_CIPHER_CTX *cipher = EVP_CIPHER_CTX_new();
-  EVP_EncryptInit(cipher, EVP_aes_128_ctr(),
-  (const unsigned char*)key, (const unsigned char *)iv);
+  const EVP_CIPHER *c;
+  switch (key_bits) {
+case 128: c = EVP_aes_128_ctr(); break;
+case 192: c = EVP_aes_192_ctr(); break;
+case 256: c = EVP_aes_256_ctr(); break;
+default: tor_assert(0); // LCOV_EXCL_LINE
+  }
+  EVP_EncryptInit(cipher, c, key, iv);
   return (aes_cnt_cipher_t *) cipher;
 }
 void
@@ -262,11 +268,11 @@ static void aes_set_iv(aes_cnt_cipher_t *cipher, const 
char *iv);
  * using the 128-bit key key and the 128-bit IV iv.
  */
 aes_cnt_cipher_t*
-aes_new_cipher(const char *key, const char *iv)
+aes_new_cipher(const uint8_t *key, const uint8_t *iv, int bits)
 {
   aes_cnt_cipher_t* result = tor_malloc_zero(sizeof(aes_cnt_cipher_t));
 
-  aes_set_key(result, key, 128);
+  aes_set_key(result, key, bits);
   aes_set_iv(result, iv);
 
   return result;
@@ -277,7 +283,7 @@ aes_new_cipher(const char *key, const char *iv)
  * the counter to 0.
  */
 static void
-aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits)
+aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
 {
   if (should_use_EVP) {
 const EVP_CIPHER *c = 0;
@@ -287,10 +293,10 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, 
int key_bits)
   case 256: c = EVP_aes_256_ecb(); break;
   default: tor_assert(0); // LCOV_EXCL_LINE
 }
-EVP_EncryptInit(>key.evp, c, (const unsigned char*)key, NULL);
+EVP_EncryptInit(>key.evp, c, key, NULL);
 cipher->using_evp = 1;
   } else {
-AES_set_encrypt_key((const unsigned char *)key, key_bits,>key.aes);
+AES_set_encrypt_key(key, key_bits,>key.aes);
 cipher->using_evp = 0;
   }
 
@@ -348,6 +354,8 @@ evp_block128_fn(const uint8_t in[16],
 void
 aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len)
 {
+  /* Note that the "128" below refers to the length of the counter,
+   * not the length of the AES key. */
   if (cipher->using_evp) {
 /* In openssl 1.0.0, there's an if'd out EVP_aes_128_ctr in evp.h.  If
  * it weren't disabled, it might be better just to use that.
@@ -374,7 +382,7 @@ aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, 
size_t len)
 /** Reset the 128-bit counter of cipher to the 16-bit big-endian value
  * in iv. */
 static void
-aes_set_iv(aes_cnt_cipher_t *cipher, const char *iv)
+aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv)
 {
 #ifdef USING_COUNTER_VARS
   cipher->counter3 = ntohl(get_uint32(iv));
diff --git a/src/common/aes.h b/src/common/aes.h
index 821fb74..1cda53f 100644
--- a/src/common/aes.h
+++ b/src/common/aes.h
@@ -15,7 +15,8 @@
 
 typedef struct aes_cnt_cipher aes_cnt_cipher_t;
 
-aes_cnt_cipher_t* aes_new_cipher(const char *key, const char *iv);
+aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
+ int key_bits);
 void aes_cipher_free(aes_cnt_cipher_t *cipher);
 void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len);
 
diff --git a/src/common/crypto.c b/src/common/crypto.c
index bf682ff..7be43d7 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -69,6 +69,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 #endif
 
 #include "torlog.h"
+#include "torint.h"
 #include "aes.h"
 #include "util.h"
 #include "container.h"
@@ -122,8 +123,8 @@ struct crypto_pk_t
 /** Key and stream information for a stream cipher. */
 struct crypto_cipher_t
 {
-  char key[CIPHER_KEY_LEN]; /**< The raw key. */
-  char iv[CIPHER_IV_LEN]; /**< The initial IV. */
+  uint8_t key[CIPHER_KEY_LEN]; /**< The raw key. */
+  uint8_t iv[CIPHER_IV_LEN]; /**< The initial IV. */
   aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES
  * encryption */
 };
@@ -561,15 +562,15 @@ crypto_cipher_new_with_iv(const char *key, const char *iv)
   env = tor_malloc_zero(sizeof(crypto_cipher_t));
 
   if (key == NULL)
-crypto_rand(env->key, CIPHER_KEY_LEN);
+crypto_rand((char*)env->key, CIPHER_KEY_LEN);
   else
 memcpy(env->key, key, CIPHER_KEY_LEN);
   

[tor-commits] [tor/master] Simplify the crypto_cipher_t interface and structure

[nickm]

commit ff116b780896bb735f887a669d87694bb6d8a964
Author: Nick Mathewson 
Date:   Fri Sep 16 10:12:30 2016 -0400

Simplify the crypto_cipher_t interface and structure

Previously, the IV and key were stored in the structure, even though
they mostly weren't needed.  The only purpose they had was to
support a seldom-used API where you could pass NULL when creating
a cipher in order to get a random key/IV, and then pull that key/IV
back out.

This saves 32 bytes per AES instance, and makes it easier to support
different key lengths.
---
 src/common/crypto.c| 45 +
 src/test/bench.c   | 16 +++-
 src/test/test_crypto.c |  9 +
 3 files changed, 33 insertions(+), 37 deletions(-)

diff --git a/src/common/crypto.c b/src/common/crypto.c
index 7be43d7..f6ee6b0 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -123,8 +123,6 @@ struct crypto_pk_t
 /** Key and stream information for a stream cipher. */
 struct crypto_cipher_t
 {
-  uint8_t key[CIPHER_KEY_LEN]; /**< The raw key. */
-  uint8_t iv[CIPHER_IV_LEN]; /**< The initial IV. */
   aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES
  * encryption */
 };
@@ -551,26 +549,18 @@ crypto_pk_free(crypto_pk_t *env)
 }
 
 /** Allocate and return a new symmetric cipher using the provided key and iv.
- * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  If you
- * provide NULL in place of either one, it is generated at random.
+ * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  Both
+ * must be provided.
  */
 crypto_cipher_t *
 crypto_cipher_new_with_iv(const char *key, const char *iv)
 {
   crypto_cipher_t *env;
+  tor_assert(key);
+  tor_assert(iv);
 
-  env = tor_malloc_zero(sizeof(crypto_cipher_t));
-
-  if (key == NULL)
-crypto_rand((char*)env->key, CIPHER_KEY_LEN);
-  else
-memcpy(env->key, key, CIPHER_KEY_LEN);
-  if (iv == NULL)
-crypto_rand((char*)env->iv, CIPHER_IV_LEN);
-  else
-memcpy(env->iv, iv, CIPHER_IV_LEN);
-
-  env->cipher = aes_new_cipher(env->key, env->iv, 128);
+  env = tor_malloc(sizeof(crypto_cipher_t));
+  env->cipher = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128);
 
   return env;
 }
@@ -1267,10 +1257,12 @@ crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
   tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
   tor_assert(tolen >= pkeylen);
 
-  cipher = crypto_cipher_new(NULL); /* generate a new key. */
+  char key[CIPHER_KEY_LEN];
+  crypto_rand(key, sizeof(key)); /* generate a new key. */
+  cipher = crypto_cipher_new(key);
 
   buf = tor_malloc(pkeylen+1);
-  memcpy(buf, cipher->key, CIPHER_KEY_LEN);
+  memcpy(buf, key, CIPHER_KEY_LEN);
   memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
 
   /* Length of symmetrically encrypted data. */
@@ -1285,6 +1277,7 @@ crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
 
   if (r<0) goto err;
   memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
   tor_free(buf);
   crypto_cipher_free(cipher);
   tor_assert(outlen+symlen < INT_MAX);
@@ -1292,6 +1285,7 @@ crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
  err:
 
   memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
   tor_free(buf);
   crypto_cipher_free(cipher);
   return -1;
@@ -1583,14 +1577,6 @@ crypto_pk_base64_decode(const char *str, size_t len)
 
 /* symmetric crypto */
 
-/** Return a pointer to the key set for the cipher in env.
- */
-const char *
-crypto_cipher_get_key(crypto_cipher_t *env)
-{
-  return (const char *)env->key;
-}
-
 /** Encrypt fromlen bytes from from using the cipher
  * env; on success, store the result to to and return 0.
  * Does not check for failure.
@@ -1660,11 +1646,14 @@ crypto_cipher_encrypt_with_iv(const char *key,
   if (tolen < fromlen + CIPHER_IV_LEN)
 return -1;
 
-  cipher = crypto_cipher_new_with_iv(key, NULL);
+  char iv[CIPHER_IV_LEN];
+  crypto_rand(iv, sizeof(iv));
+  cipher = crypto_cipher_new_with_iv(key, iv);
 
-  memcpy(to, cipher->iv, CIPHER_IV_LEN);
+  memcpy(to, iv, CIPHER_IV_LEN);
   crypto_cipher_encrypt(cipher, to+CIPHER_IV_LEN, from, fromlen);
   crypto_cipher_free(cipher);
+  memwipe(iv, 0, sizeof(iv));
   return (int)(fromlen + CIPHER_IV_LEN);
 }
 
diff --git a/src/test/bench.c b/src/test/bench.c
index f373019..30984fd 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -90,7 +90,9 @@ bench_aes(void)
   uint64_t start, end;
   const int bytes_per_iter = (1<<24);
   reset_perftime();
-  c = crypto_cipher_new(NULL);
+  char key[CIPHER_KEY_LEN];
+  crypto_rand(key, sizeof(key));
+  c = crypto_cipher_new(key);
 
   for (len = 1; len <= 8192; len *= 2) {
 int iters = bytes_per_iter / len;
@@ -328,8 +330,9 @@ bench_cell_aes(void)
   char *b = tor_malloc(len+max_misalign);
   crypto_cipher_t *c;
   int i, misalign;
-
-  c = crypto_cipher_new(NULL);
+  char key[CIPHER_KEY_LEN];
+  

[tor-commits] [tor/master] Remove a needless level of indirection from crypto_cipher_t

[nickm]

commit 83129031b1a1a3c719810d30df0e3ec6fa320661
Author: Nick Mathewson 
Date:   Fri Sep 16 10:18:02 2016 -0400

Remove a needless level of indirection from crypto_cipher_t

Now that crypto_cipher_t only contains a pointer, it no longer
has any reason for an independent existence.
---
 src/common/crypto.c | 23 ++-
 src/common/crypto.h |  2 +-
 2 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/src/common/crypto.c b/src/common/crypto.c
index f6ee6b0..fb77340 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -120,13 +120,6 @@ struct crypto_pk_t
   RSA *key; /**< The key itself */
 };
 
-/** Key and stream information for a stream cipher. */
-struct crypto_cipher_t
-{
-  aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES
- * encryption */
-};
-
 /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake
  * while we're waiting for the second.*/
 struct crypto_dh_t {
@@ -559,8 +552,7 @@ crypto_cipher_new_with_iv(const char *key, const char *iv)
   tor_assert(key);
   tor_assert(iv);
 
-  env = tor_malloc(sizeof(crypto_cipher_t));
-  env->cipher = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128);
+  env = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128);
 
   return env;
 }
@@ -583,10 +575,7 @@ crypto_cipher_free(crypto_cipher_t *env)
   if (!env)
 return;
 
-  tor_assert(env->cipher);
-  aes_cipher_free(env->cipher);
-  memwipe(env, 0, sizeof(crypto_cipher_t));
-  tor_free(env);
+  aes_cipher_free(env);
 }
 
 /* public key crypto */
@@ -1586,14 +1575,14 @@ crypto_cipher_encrypt(crypto_cipher_t *env, char *to,
   const char *from, size_t fromlen)
 {
   tor_assert(env);
-  tor_assert(env->cipher);
+  tor_assert(env);
   tor_assert(from);
   tor_assert(fromlen);
   tor_assert(to);
   tor_assert(fromlen < SIZE_T_CEILING);
 
   memcpy(to, from, fromlen);
-  aes_crypt_inplace(env->cipher, to, fromlen);
+  aes_crypt_inplace(env, to, fromlen);
   return 0;
 }
 
@@ -1611,7 +1600,7 @@ crypto_cipher_decrypt(crypto_cipher_t *env, char *to,
   tor_assert(fromlen < SIZE_T_CEILING);
 
   memcpy(to, from, fromlen);
-  aes_crypt_inplace(env->cipher, to, fromlen);
+  aes_crypt_inplace(env, to, fromlen);
   return 0;
 }
 
@@ -1622,7 +1611,7 @@ void
 crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *buf, size_t len)
 {
   tor_assert(len < SIZE_T_CEILING);
-  aes_crypt_inplace(env->cipher, buf, len);
+  aes_crypt_inplace(env, buf, len);
 }
 
 /** Encrypt fromlen bytes (at least 1) from from with the key in
diff --git a/src/common/crypto.h b/src/common/crypto.h
index 2d1155e..e60cf34 100644
--- a/src/common/crypto.h
+++ b/src/common/crypto.h
@@ -117,7 +117,7 @@ typedef struct {
 } common_digests_t;
 
 typedef struct crypto_pk_t crypto_pk_t;
-typedef struct crypto_cipher_t crypto_cipher_t;
+typedef struct aes_cnt_cipher crypto_cipher_t;
 typedef struct crypto_digest_t crypto_digest_t;
 typedef struct crypto_xof_t crypto_xof_t;
 typedef struct crypto_dh_t crypto_dh_t;



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] Merge remote-tracking branch 'public/aes256'

[nickm]

commit af01e8211e08fd418365f66c79a04e0db5a74aaa
Merge: 144bd86 6cb9c2c
Author: Nick Mathewson 
Date:   Mon Sep 19 14:22:27 2016 -0400

Merge remote-tracking branch 'public/aes256'

 src/common/aes.c   | 28 +--
 src/common/aes.h   |  3 +-
 src/common/crypto.c| 98 +-
 src/common/crypto.h|  6 +++-
 src/test/bench.c   | 16 ++---
 src/test/test_crypto.c | 71 ++--
 6 files changed, 137 insertions(+), 85 deletions(-)

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] Document the default PathsNeededToBuildCircuits value

[nickm]

commit 42a74f707c3031dab2aa21983040f58ba7b190e4
Author: teor 
Date:   Fri Sep 9 11:20:20 2016 +1000

Document the default PathsNeededToBuildCircuits value

... when the directory authorities don't set min_paths_for_circs_pct.

Fixes bug 20117; bugfix on 02c320916e02 in tor-0.2.4.10-alpha.
Reported by Jesse V.
---
 changes/bug20117 | 6 ++
 doc/tor.1.txt| 5 +++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/changes/bug20117 b/changes/bug20117
new file mode 100644
index 000..6532260
--- /dev/null
+++ b/changes/bug20117
@@ -0,0 +1,6 @@
+  o Minor bugfixes (documentation):
+- Document the default PathsNeededToBuildCircuits value that's
+  used by clients when the directory authorities don't set
+  min_paths_for_circs_pct.
+  Fixes bug 20117; bugfix on 02c320916e02 in tor-0.2.4.10-alpha.
+  Patch by teor, reported by Jesse V.
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index 1856592..f53ff94 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -1558,8 +1558,9 @@ The following options are useful only for clients (that 
is, if
 that fraction of possible paths. Note that setting this option too low
 can make your Tor client less anonymous, and setting it too high can
 prevent your Tor client from bootstrapping.  If this option is negative,
-Tor will use a default value chosen by the directory
-authorities. (Default: -1.)
+Tor will use a default value chosen by the directory authorities. If the
+directory authorities do not choose a value, Tor will default to 0.6.
+(Default: -1.)
 
 [[ClientBootstrapConsensusAuthorityDownloadSchedule]] 
**ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__::
 Schedule for when clients should download consensuses from authorities



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [tor/master] Merge remote-tracking branch 'teor/bug20117'

[nickm]

commit 144bd86570ee2afc4efd8ecd7c59a65973452064
Merge: b08ddb6 42a74f7
Author: Nick Mathewson 
Date:   Mon Sep 19 14:21:12 2016 -0400

Merge remote-tracking branch 'teor/bug20117'

 changes/bug20117 | 6 ++
 doc/tor.1.txt| 5 +++--
 2 files changed, 9 insertions(+), 2 deletions(-)

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [collector/master] Sanitize TCP ports in bridge descriptors.

[karsten]

commit ecb053899eb965c2778cf05479c26549d67f7956
Author: Karsten Loesing 
Date:   Fri Jun 10 13:28:42 2016 +0200

Sanitize TCP ports in bridge descriptors.

Implements #19317.
---
 CHANGELOG.md   |  2 +
 .../bridgedescs/SanitizedBridgesWriter.java| 81 +-
 src/main/webapp/index.html | 46 ++--
 3 files changed, 107 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24206ee..e17abad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@
- Add support for Bifroest's bridge descriptor tarballs.
- Use a shutdown hook that gives currently running modules up to 10
  minutes to finish properly, rather than killing them immediately.
+   - Replace TCP ports with hashes in @type bridge-network-status 1.1
+ and @type bridge-server-descriptor 1.2.
 
  * Minor changes
- Remove quotes around base URL in index.json.
diff --git 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index b787f78..b61cd30 100644
--- 
a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ 
b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -138,7 +138,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
 while ((line = br.readLine()) != null) {
   String[] parts = line.split(",");
   if ((line.length() != ("-MM,".length() + 31 * 2)
-  && line.length() != ("-MM,".length() + 50 * 2))
+  && line.length() != ("-MM,".length() + 50 * 2)
+  && line.length() != ("-MM,".length() + 83 * 2))
   || parts.length != 2) {
 logger.warn("Invalid line in bridge-ip-secrets file "
 + "starting with '" + line.substring(0, 7) + "'! "
@@ -218,8 +219,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
   scrubbedAddressPart = this.scrubIpv4Address(addressPart,
   fingerprintBytes, published);
 }
+String scrubbedPort = this.scrubTcpPort(portPart, fingerprintBytes,
+published);
 return (scrubbedAddressPart == null ? null :
-  scrubbedAddressPart + ":" + portPart);
+  scrubbedAddressPart + ":" + scrubbedPort);
   }
 
   private String scrubIpv4Address(String address, byte[] fingerprintBytes,
@@ -334,14 +337,42 @@ public class SanitizedBridgesWriter extends CollecTorMain 
{
 return sb.toString();
   }
 
+  private String scrubTcpPort(String portString, byte[] fingerprintBytes,
+  String published) throws IOException {
+if (portString.equals("0")) {
+  return "0";
+} else if (this.replaceIpAddressesWithHashes) {
+  if (this.persistenceProblemWithSecrets) {
+/* There's a persistence problem, so we shouldn't scrub more TCP
+ * ports in this execution. */
+return null;
+  }
+  byte[] hashInput = new byte[2 + 20 + 33];
+  int portNumber = Integer.parseInt(portString);
+  hashInput[0] = (byte) (portNumber >> 8);
+  hashInput[1] = (byte) portNumber;
+  System.arraycopy(fingerprintBytes, 0, hashInput, 2, 20);
+  String month = published.substring(0, "-MM".length());
+  byte[] secret = this.getSecretForMonth(month);
+  System.arraycopy(secret, 50, hashInput, 22, 33);
+  byte[] hashOutput = DigestUtils.sha256(hashInput);
+  int hashedPort = hashOutput[0] & 0xFF) << 8)
+  | (hashOutput[1] & 0xFF)) >> 2) | 0xC000;
+  return String.valueOf(hashedPort);
+} else {
+  return "1";
+}
+  }
+
   private byte[] getSecretForMonth(String month) throws IOException {
 if (!this.secretsForHashingIpAddresses.containsKey(month)
-|| this.secretsForHashingIpAddresses.get(month).length == 31) {
-  byte[] secret = new byte[50];
+|| this.secretsForHashingIpAddresses.get(month).length < 83) {
+  byte[] secret = new byte[83];
   this.secureRandom.nextBytes(secret);
   if (this.secretsForHashingIpAddresses.containsKey(month)) {
 System.arraycopy(this.secretsForHashingIpAddresses.get(month), 0,
-secret, 0, 31);
+secret, 0,
+this.secretsForHashingIpAddresses.get(month).length);
   }
   if (month.compareTo(
   this.bridgeSanitizingCutOffTimestamp) < 0) {
@@ -362,8 +393,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
   bw.close();
 } catch (IOException e) {
   logger.warn("Could not store new secret "
-  + "to disk! Not calculating any IP address hashes in "
-  + "this execution!", e);
+  + "to disk! Not calculating any IP address or TCP port "
+  + "hashes in this execution!", e);
   this.persistenceProblemWithSecrets = true;
   throw new 

[tor-commits] [collector/master] Split up bridge descriptor tarballs into one per type.

[karsten]

commit f608c94c7f731241bf7ee8e627ca1da98c23d858
Author: Karsten Loesing 
Date:   Sun Sep 18 16:09:52 2016 +0200

Split up bridge descriptor tarballs into one per type.
---
 CHANGELOG.md  |  4 
 src/main/resources/create-tarballs.sh | 26 --
 src/main/webapp/index.html| 19 +++
 3 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e17abad..99b2918 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,10 @@
  minutes to finish properly, rather than killing them immediately.
- Replace TCP ports with hashes in @type bridge-network-status 1.1
  and @type bridge-server-descriptor 1.2.
+   - Split up bridge descriptor tarballs into one tarball per month
+ and descriptor type: bridge-statuses--MM.tar.xz,
+ bridge-server-descriptors--MM.tar.xz and
+ bridge-extra-infos--MM.tar.xz.
 
  * Minor changes
- Remove quotes around base URL in index.json.
diff --git a/src/main/resources/create-tarballs.sh 
b/src/main/resources/create-tarballs.sh
index cc75219..4f2c6eb 100755
--- a/src/main/resources/create-tarballs.sh
+++ b/src/main/resources/create-tarballs.sh
@@ -56,8 +56,12 @@ TARBALLS=(
   server-descriptors-$YEARTWO-$MONTHTWO
   extra-infos-$YEARONE-$MONTHONE
   extra-infos-$YEARTWO-$MONTHTWO
-  bridge-descriptors-$YEARONE-$MONTHONE
-  bridge-descriptors-$YEARTWO-$MONTHTWO
+  bridge-statuses-$YEARONE-$MONTHONE
+  bridge-statuses-$YEARTWO-$MONTHTWO
+  bridge-server-descriptors-$YEARONE-$MONTHONE
+  bridge-server-descriptors-$YEARTWO-$MONTHTWO
+  bridge-extra-infos-$YEARONE-$MONTHONE
+  bridge-extra-infos-$YEARTWO-$MONTHTWO
 )
 TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))
 
@@ -77,8 +81,12 @@ DIRECTORIES=(
   $OUTDIR/relay-descriptors/server-descriptor/$YEARTWO/$MONTHTWO/
   $OUTDIR/relay-descriptors/extra-info/$YEARONE/$MONTHONE/
   $OUTDIR/relay-descriptors/extra-info/$YEARTWO/$MONTHTWO/
-  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/
-  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/statuses/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/statuses/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/server-descriptors/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/server-descriptors/
+  $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/extra-infos/
+  $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/
 )
 DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))
 
@@ -109,8 +117,14 @@ cd $CURRENTPATH
 
 echo `date` "Finished tarball creation.  Starting symlink-update ..."
 
-mkdir -p $ARCHIVEDIR/bridge-descriptors/
-ln -f -s -t $ARCHIVEDIR/bridge-descriptors/ 
$TARBALLTARGETDIR/bridge-descriptors-20??-??.tar.xz
+mkdir -p $ARCHIVEDIR/bridge-descriptors/statuses/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/statuses/ 
$TARBALLTARGETDIR/bridge-statuses-20??-??.tar.xz
+
+mkdir -p $ARCHIVEDIR/bridge-descriptors/server-descriptors/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/server-descriptors/ 
$TARBALLTARGETDIR/bridge-server-descriptors-20??-??.tar.xz
+
+mkdir -p $ARCHIVEDIR/bridge-descriptors/extra-infos/
+ln -f -s -t $ARCHIVEDIR/bridge-descriptors/extra-infos/ 
$TARBALLTARGETDIR/bridge-extra-infos-20??-??.tar.xz
 
 mkdir -p $ARCHIVEDIR/bridge-pool-assignments/
 ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ 
$TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz
diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html
index bbc70f2..7ed9145 100644
--- a/src/main/webapp/index.html
+++ b/src/main/webapp/index.html
@@ -168,21 +168,21 @@
   Bridge Network Statuses
   @type bridge-network-status 1.1
   recent
-  archive
+  archive
   format
 
 
   Bridge Server Descriptors
   @type bridge-server-descriptor 1.2
   recent
-  archive
+  archive
   format
 
 
   Bridge Extra-info Descriptors
   @type bridge-extra-info 1.3
   recent
-  archive
+  archive
   format
 
 
@@ -500,8 +500,6 @@ descriptor including all signatures.
 Sanitized bridge network statuses are similar to version 2 relay network
 statuses, but with only a published line in the header and
 without any lines in the footer.
-The bridge descriptor archive tarballs contain all bridge
-descriptors of a given month, not just network statuses.
 The format has changed over time to accomodate changes to the sanitizing
 process, with earlier versions being:
 
@@ -523,10 +521,8 @@ ports.
 
 Bridge server descriptors follow the same format as relay server
 descriptors, except for the sanitizing steps described above.
-The bridge descriptor archive tarballs contain all bridge
-descriptors of a given month, not just server descriptors.
-These tarballs contain one descriptor per file, whereas recently published 
bridge descriptor
-files
+The bridge server descriptor archive tarballs contain one descriptor per
+file, whereas recently published bridge server descriptor files
 contain all descriptors collected 

[tor-commits] [ooni-probe/master] ooni/agent/scheduler.py: fix easter egg date

[art]

commit 1b9cb5e63687ca41abadb38282717b63620c719b
Author: Simone Basso 
Date:   Mon Sep 19 12:43:07 2016 +0200

ooni/agent/scheduler.py: fix easter egg date
---
 ooni/agent/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index e8fef53..be141eb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -53,7 +53,7 @@ class FileSystemlockAndMutex(object):
 
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
-CANARY_DATE = datetime(1957, 8, 4, tzinfo=tz.tzutc())
+CANARY_DATE = datetime(1957, 10, 4, tzinfo=tz.tzutc())
 
 
 class DidNotRun(Exception):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ui/web/server.py: pass binary, not unicode cookies

[art]

commit 21548e9f0f53100336ef4f22d423f5f728a117e5
Author: Simone Basso 
Date:   Mon Sep 19 11:04:53 2016 +0200

ui/web/server.py: pass binary, not unicode cookies

It seems that Twisted has added a function enforcing binary
data (not unicode, or str) when sending to the network.

Thus, make sure that we pass cookies as binary (more specifically
ascii encoded data) to avoid a Twisted error.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index eec82d3..385bf18 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(u'XSRF-TOKEN')
+token_cookie = request.getCookie(b'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(u'XSRF-TOKEN',
+request.addCookie(b'XSRF-TOKEN',
   instance._xsrf_token,
-  path=u'/')
+  path=b'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = ''.join([rng.choice(token_space)
+self._xsrf_token = b''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add changelog entry for 2.0.0-rc.3

[art]

commit 77c2e0f7f2c1df0d7490d560103ba241c5502c87
Author: Arturo Filastò 
Date:   Mon Sep 19 14:11:00 2016 +0200

Add changelog entry for 2.0.0-rc.3
---
 ChangeLog.rst | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 3c0ecd8..f847675 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,11 @@
 Changelog
 =
 
+v2.0.0-rc.3 (Mon, 19 Sep 2016)
+--
+
+Bugfixing and code cleanup
+
 v2.0.0-rc.2 (Tue, 13 Sep 2016)
 --
 

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Retry 3 times the http_requests integration tests.

[art]

commit 7be2fc5571c9a0c7fa65de70381819f9f458e6fe
Author: Arturo Filastò 
Date:   Fri Sep 16 19:19:31 2016 +0200

Retry 3 times the http_requests integration tests.

Hopefully this will overcome the transient failures.
---
 ooni/tests/test_oonicli.py | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 4a6edf9..05bc273 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -111,6 +111,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def test_http_requests(self):
+retries = 3
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry
@@ -119,10 +120,14 @@ class TestRunDirector(ConfigTestCase):
 assert 'factor' in entry
 assert 'headers_diff' in entry
 assert 'headers_match' in entry
-
-yield self.run_helper('blocking/http_requests',
-  ['-u', 'http://torproject.org/'],
-  verify_function)
+while retries > 0:
+try:
+yield self.run_helper('blocking/http_requests',
+['-u', 'http://torproject.org/'],
+verify_function)
+break
+except:
+retries -= 1
 
 @defer.inlineCallbacks
 def test_http_requests_with_file(self):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Strip the filepath from input_store listing

[art]

commit baa505da2dcd2cf69e58fe212a2bc8478cf99617
Author: Arturo Filastò 
Date:   Sun Sep 18 16:41:38 2016 +0200

Strip the filepath from input_store listing
---
 ooni/deck/store.py| 4 ++--
 ooni/ui/web/server.py | 6 --
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index f30b0d6..3e20e01 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -115,7 +115,7 @@ class InputStore(object):
 if self._cache_stale:
 self._update_cache()
 try:
-input_desc = self._cache[input_id]
+input_desc = deepcopy(self._cache[input_id])
 except KeyError:
 raise InputNotFound(input_id)
 return input_desc
@@ -190,7 +190,7 @@ class DeckStore(object):
 if self._cache_stale:
 self._update_cache()
 try:
-return self._cache[deck_id]
+return deepcopy(self._cache[deck_id])
 except KeyError:
 raise DeckNotFound(deck_id)
 
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index cee42ce..eec82d3 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -474,7 +474,7 @@ class WebUIAPI(object):
 
 except errors.InsufficientPrivileges:
 raise WebUIError(
-400, 'Insufficient priviledges'
+400, 'Insufficient privileges'
 )
 except:
 raise WebUIError(
@@ -511,8 +511,10 @@ class WebUIAPI(object):
 @xsrf_protect(check=False)
 @requires_true(attrs=['_is_initialized'])
 def api_input_details(self, request, input_id):
+input_desc = self.director.input_store.get(input_id)
+input_desc.pop('filepath')
 return self.render_json(
-self.director.input_store.get(input_id), request
+input_desc, request
 )
 
 @app.route('/api/measurement', methods=["GET"])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] First kill ooniprobe-agent with SIGTERM and then hit it harder with SIGKILL

[art]

commit 16fa7de0af33194376ea1da6a8813ffe85d18a81
Author: Arturo Filastò 
Date:   Fri Sep 16 11:57:32 2016 +0200

First kill ooniprobe-agent with SIGTERM and then hit it harder with SIGKILL
---
 ooni/scripts/ooniprobe_agent.py | 20 
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 479d230..4c97f5e 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -128,12 +128,15 @@ def stop_agent():
 pid = open(pidfile, "r").read()
 pid = int(pid)
 try:
-os.kill(pid, signal.SIGKILL)
+os.kill(pid, signal.SIGTERM)
 except OSError as ose:
 if ose.errno == errno.ESRCH:
 print("No process was running. Cleaning up.")
 # the process didn't exist, so wipe the pid file
-os.remove(pidfile)
+try:
+os.remove(pidfile)
+except EnvironmentError as exc:
+print("Failed to delete the pidfile {0}".format(exc))
 return 2
 elif ose.errno == errno.EPERM:
 # The process is owned by root. We assume it's running
@@ -152,7 +155,7 @@ def stop_agent():
 while True:
 # poll once per second until we see the process is no longer running
 try:
-os.kill(pid, 0)
+os.kill(pid, signal.SIG_DFL)
 except OSError:
 print("process %d is dead" % pid)
 return
@@ -161,7 +164,16 @@ def stop_agent():
 if first_time:
 print("It looks like pid %d is still running "
   "after %d seconds" % (pid, (time.time() - start)))
-print("I will keep watching it until you interrupt me.")
+print("Sending a SIGKILL and waiting for it to terminate "
+  "until you kill me.")
+try:
+os.kill(pid, signal.SIGKILL)
+except OSError as ose:
+# Race condition check. It could have dies already. If
+# so we are happy.
+if ose.errno == errno.ESRCH:
+print("process %d is dead" % pid)
+return
 wait = 10
 first_time = False
 else:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The version numbers of oonideckgen, oonireport and ooniresources are now the same of ooniprobe

[art]

commit 9f92dc3eefef065ef0be7b577950e2c916f2cc47
Author: Arturo Filastò 
Date:   Fri Sep 16 11:57:02 2016 +0200

The version numbers of oonideckgen, oonireport and ooniresources are now 
the same of ooniprobe
---
 ooni/scripts/oonideckgen.py   | 4 +---
 ooni/scripts/oonireport.py| 3 +--
 ooni/scripts/ooniresources.py | 3 ++-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 4ad0f59..ba27ffe 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -11,10 +11,8 @@ from ooni.otime import prettyDateNowUTC
 from ooni import errors
 from ooni.geoip import probe_ip
 from ooni.resources import check_for_update
-from ooni.settings import config
 from ooni.deck import NGDeck
-
-__version__ = "1.0.0"
+from ooni import __version__
 
 class Options(usage.Options):
 synopsis = """%s [options]
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index cd9b244..f7896aa 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -14,8 +14,7 @@ from ooni.reporter import OONIBReporter, OONIBReportLog
 from ooni.utils import log
 from ooni.settings import config
 from ooni.backend_client import BouncerClient, CollectorClient
-
-__version__ = "0.1.0"
+from ooni import __version__
 
 @defer.inlineCallbacks
 def lookup_collector_client(report_header, bouncer):
diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py
index 8e77431..2ce5c9e 100644
--- a/ooni/scripts/ooniresources.py
+++ b/ooni/scripts/ooniresources.py
@@ -1,6 +1,7 @@
 import sys
 
 from twisted.python import usage
+from ooni import __version__
 
 class Options(usage.Options):
 synopsis = """%s
@@ -17,7 +18,7 @@ class Options(usage.Options):
 optParameters = []
 
 def opt_version(self):
-print("ooniresources version: 0.2.0")
+print("ooniresources version: %s" % __version__)
 sys.exit(0)
 
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Various code style fixes suggested by @bassosimone

[art]

commit 3483c8005ecf8eb0c856f04913f6132f547f2d84
Author: Arturo Filastò 
Date:   Fri Sep 16 11:56:38 2016 +0200

Various code style fixes suggested by @bassosimone
---
 ooni/reporter.py|  6 +++---
 ooni/resources.py   |  4 
 ooni/scripts/ooniprobe.py   | 10 ++
 ooni/scripts/ooniprobe_agent.py | 12 +---
 ooni/scripts/ooniresources.py   |  3 ++-
 5 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index 951602f..ab32c54 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -221,7 +221,8 @@ class NJSONReporter(OReporter):
 OReporter.__init__(self, test_details)
 
 def _writeln(self, line):
-self._write("%s\n" % line)
+self._write(line)
+self._write("\n")
 
 def _write(self, data):
 if not self._stream:
@@ -249,8 +250,7 @@ class NJSONReporter(OReporter):
 'test_keys': e
 }
 report_entry.update(self.testDetails)
-self._write(json.dumps(report_entry))
-self._write("\n")
+self._writeln(json.dumps(report_entry))
 
 def createReport(self):
 self._stream = open(self.report_path, 'w+')
diff --git a/ooni/resources.py b/ooni/resources.py
index ac0f967..edb7781 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -91,6 +91,10 @@ def check_for_update(country_code=None):
 Checks if we need to update the resources.
 If the country_code is specified then only the resources for that
 country will be updated/downloaded.
+
+XXX we currently don't check the shasum of resources although this is
+included inside of the manifest.
+This should probably be done once we have signing of resources.
 :return: the latest version.
 """
 temporary_files = []
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index d67dd80..c8ffee2 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -12,10 +12,12 @@ def ooniprobe(reactor):
 check_incoherences=True)
 if global_options['queue']:
 return runWithDaemonDirector(global_options)
-elif global_options['initialize']:
+
+if global_options['initialize']:
 initializeOoniprobe(global_options)
 return defer.succeed(None)
-elif global_options['web-ui']:
+
+if global_options['web-ui']:
 from ooni.scripts.ooniprobe_agent import WEB_UI_URL
 from ooni.scripts.ooniprobe_agent import status_agent, start_agent
 if status_agent() != 0:
@@ -25,8 +27,8 @@ def ooniprobe(reactor):
 print("Started ooniprobe-agent")
 webbrowser.open_new(WEB_UI_URL)
 return defer.succeed(None)
-else:
-return runWithDirector(global_options)
+
+return runWithDirector(global_options)
 
 def run():
 task.react(ooniprobe)
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index c455f9b..479d230 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import sys
 import time
 import errno
 import signal
@@ -65,8 +66,9 @@ def start_agent(options=None):
 twistd_args.append("StartOoniprobeAgent")
 try:
 twistd_config.parseOptions(twistd_args)
-except usage.error, ue:
+except usage.error as ue:
 print("ooniprobe: usage error from twistd: {}\n".format(ue))
+sys.exit(1)
 twistd_config.loadedPlugins = {
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
@@ -103,12 +105,8 @@ def get_running_pidfile():
 pid = open(pidfile, "r").read()
 pid = int(pid)
 if is_process_running(pid):
-running_pidfile = pidfile
-else:
-continue
-if running_pidfile is None:
-raise NotRunning
-return running_pidfile
+return pidfile
+raise NotRunning
 
 def status_agent():
 try:
diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py
index 5fdbbf0..8e77431 100644
--- a/ooni/scripts/ooniresources.py
+++ b/ooni/scripts/ooniresources.py
@@ -30,5 +30,6 @@ def run():
 print "%s: Try --help for usage details." % (sys.argv[0])
 sys.exit(1)
 
-print("WARNING: Usage of this script is deprecated.")
+print("WARNING: Usage of this script is deprecated. We will not do "
+  "anything.")
 sys.exit(0)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump the timeout value for test_oonicli.

[art]

commit 4fd30aab622dcd9dc50311dd8e1ccf58cdafaa87
Author: Arturo Filastò 
Date:   Fri Sep 16 18:23:40 2016 +0200

Bump the timeout value for test_oonicli.

Travis has gotten slower or our tests have gotten slower, either way let's
increase this to avoid all the annoying transient travis failures.
---
 ooni/tests/test_oonicli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 8ad8a36..4a6edf9 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -59,7 +59,7 @@ tor:
 
 
 class TestRunDirector(ConfigTestCase):
-timeout = 220
+timeout = 420
 
 def setUp(self):
 super(TestRunDirector, self).setUp()



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Refactor the pattern for directory creation into a function

[art]

commit ba8558514aabc963782688ac694daa196c53dc8d
Author: Arturo Filastò 
Date:   Fri Sep 16 12:10:20 2016 +0200

Refactor the pattern for directory creation into a function
---
 ooni/deck/store.py  | 14 --
 ooni/resources.py   | 16 +---
 ooni/scripts/oonideckgen.py |  8 ++--
 ooni/utils/__init__.py  | 13 +
 ooni/utils/log.py   |  9 ++---
 ooni/utils/onion.py |  8 +++-
 6 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index bd852c7..bf1eb7f 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -6,6 +6,7 @@ from copy import deepcopy
 from twisted.internet import defer
 from twisted.python.filepath import FilePath
 
+from ooni.utils import mkdir_p
 from ooni.deck.deck import NGDeck
 from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
@@ -80,16 +81,9 @@ class InputStore(object):
 self.path = FilePath(config.inputs_directory)
 self.resources = FilePath(config.resources_directory)
 
-try:
-self.path.child("descriptors").makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
-try:
-self.path.child("data").makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+mkdir_p(self.path.child("descriptors").path)
+mkdir_p(self.path.child("data").path)
+
 yield self.update_url_lists(country_code)
 
 @defer.inlineCallbacks
diff --git a/ooni/resources.py b/ooni/resources.py
index edb7781..1830d42 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -5,7 +5,7 @@ from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
-from ooni.utils import log, gunzip, rename
+from ooni.utils import log, gunzip, rename, mkdir_p
 from ooni.settings import config
 
 # Disable logs of HTTPClientFactory
@@ -107,11 +107,7 @@ def check_for_update(country_code=None):
 latest_version = yield get_latest_version()
 
 resources_dir = FilePath(config.resources_directory)
-try:
-resources_dir.makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+mkdir_p(resources_dir.path)
 current_manifest = resources_dir.child("manifest.json")
 
 if current_manifest.exists():
@@ -153,11 +149,9 @@ def check_for_update(country_code=None):
 filename = filename[:-3]
 gzipped = True
 dst_file = resources_dir.child(pre_path).child(filename)
-try:
-dst_file.parent().makedirs()
-except OSError as e:
-if not e.errno == errno.EEXIST:
-raise
+
+mkdir_p(dst_file.parent().path)
+
 src_file = dst_file.temporarySibling()
 src_file.alwaysCreate = 0
 
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index ba27ffe..89541d7 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -1,12 +1,12 @@
 from __future__ import print_function
 
-import errno
 import os
 import sys
 
 from twisted.internet import defer, task
 from twisted.python import usage
 
+from ooni.utils import mkdir_p
 from ooni.otime import prettyDateNowUTC
 from ooni import errors
 from ooni.geoip import probe_ip
@@ -120,11 +120,7 @@ def oonideckgen(reactor):
 
 options['country-code'] = options['country-code'].lower()
 
-try:
-os.makedirs(os.path.dirname(options['output']))
-except OSError as exception:
-if exception.errno != errno.EEXIST:
-raise
+mkdir_p(os.path.dirname(options['output']))
 
 generate_deck(options)
 
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index a894daf..99f1985 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -175,3 +175,16 @@ def is_process_running(pid):
 else:
 raise
 return running
+
+def mkdir_p(path):
+"""
+Like makedirs, but it also ignores EEXIST errors, unless it exists but
+isn't a directory.
+"""
+try:
+os.makedirs(path)
+except OSError as ose:
+if ose.errno != errno.EEXIST:
+raise
+if not os.path.isdir(path):
+raise
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 93e6b31..f20fdce 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -7,6 +7,7 @@ import logging
 from twisted.python import log as tw_log
 from twisted.python.logfile import DailyLogFile
 
+from ooni.utils import mkdir_p
 from ooni import otime
 
 # Get rid of the annoying "No route found for
@@ -140,13 +141,7 @@ class OONILogger(object):
 log_folder = config.running_path
 logfile = os.path.join(log_folder, "ooniprobe.log")
 
-try:
-   

[tor-commits] [ooni-probe/master] Merge pull request #582 from TheTorProject/v2.0.0-alpha

[art]

commit 7d5f430becd2372d71c8e5839d32244121dbbe80
Merge: da00195 b6b03ac
Author: Simone Basso 
Date:   Mon Sep 19 13:56:08 2016 +0200

Merge pull request #582 from TheTorProject/v2.0.0-alpha

V2.0.0 alpha

 .coveragerc| 2 +-
 .gitignore | 4 +
 ChangeLog.rst  |18 +
 MANIFEST.in| 8 +-
 Vagrantfile|33 +-
 bin/Makefile   |81 -
 bin/oonid  | 7 -
 bin/oonideckgen|37 -
 bin/ooniprobe  |24 -
 bin/ooniprobe-dev  |10 -
 bin/oonireport |38 -
 bin/ooniresources  |35 -
 bin/test/ooni/__init__.py  | 1 -
 data/configs/lepidopter-ooniprobe.conf |75 -
 data/configs/lepidopter-oonireport.conf|69 -
 data/decks/complete.deck   |72 -
 data/decks/complete_no_root.deck   |60 -
 data/decks/fast.deck   |36 -
 data/decks/fast_no_root.deck   |24 -
 data/decks/mlab.deck   |12 -
 data/decks/web-full.yaml   |23 +
 data/decks/web-no-invalid.yaml |18 +
 data/inputs/.gitignore | 2 -
 data/lepidopter-update.py  |   385 +
 data/ooniprobe.conf.sample | 7 +-
 data/resources/README  | 2 -
 data/ui/.bowerrc   | 5 -
 data/ui/app/index.html |39 -
 data/ui/app/libs/angular-resource/README.md| 4 -
 .../app/libs/angular-resource/angular-resource.js  |   445 -
 data/ui/app/libs/angular-resource/component.json   |17 -
 data/ui/app/libs/angular/angular.js| 14733 ---
 data/ui/app/libs/angular/component.json|14 -
 data/ui/app/libs/bootstrap/component.json  | 9 -
 .../libs/bootstrap/css/bootstrap-responsive.css|  1109 --
 data/ui/app/libs/bootstrap/css/bootstrap.css   |  6158 
 .../bootstrap/img/glyphicons-halflings-white.png   |   Bin 8777 -> 0 bytes
 .../libs/bootstrap/img/glyphicons-halflings.png|   Bin 12799 -> 0 bytes
 data/ui/app/libs/bootstrap/js/bootstrap.js |  2276 ---
 data/ui/app/libs/jquery/component.json |14 -
 data/ui/app/libs/jquery/composer.json  |23 -
 data/ui/app/libs/jquery/jquery.js  |  9472 
 data/ui/app/libs/ng-upload/component.json  |23 -
 data/ui/app/libs/ng-upload/ng-upload.js|   107 -
 data/ui/app/scripts/app.js |30 -
 data/ui/app/scripts/controllers.js |96 -
 data/ui/app/scripts/directives.js  | 5 -
 data/ui/app/scripts/filters.js | 5 -
 data/ui/app/scripts/services.js|27 -
 data/ui/app/styles/app.css |21 -
 data/ui/app/views/inputs.html  |31 -
 data/ui/app/views/settings.html| 0
 data/ui/app/views/sidebar.html |10 -
 data/ui/app/views/test.html|48 -
 data/ui/component.json | 9 -
 ooni/__init__.py   |13 +-
 ooni/agent/__init__.py | 0
 ooni/agent/agent.py|28 +
 ooni/agent/scheduler.py|   432 +
 ooni/api/__init__.py   | 0
 ooni/api/spec.py   |   253 -
 ooni/backend_client.py |   117 +-
 ooni/constants.py  | 4 +-
 ooni/contrib/__init__.py   | 1 +
 ooni/contrib/croniter.py   |   430 +
 ooni/contrib/dateutil/__init__.py  | 0
 ooni/contrib/dateutil/relativedelta.py |   539 +
 ooni/contrib/dateutil/tz/__init__.py   | 4 +
 ooni/contrib/dateutil/tz/_common.py|   100 +
 ooni/contrib/dateutil/tz/tz.py |  1339 ++
 ooni/contrib/dateutil/tz/win.py|   354 +
 ooni/deck.py   |   413 -
 ooni/deck/__init__.py  | 1 +
 ooni/deck/backend.py   |   191 +
 ooni/deck/deck.py  |   411 +
 ooni/deck/legacy.py  

[tor-commits] [ooni-probe/master] Skip tests for http_requests integration tests

[art]

commit a89d9124caaac15dd54af44975e6029821c41f78
Author: Arturo Filastò 
Date:   Mon Sep 19 12:43:55 2016 +0200

Skip tests for http_requests integration tests
---
 ooni/tests/test_oonicli.py | 14 +-
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 05bc273..896a4b0 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -111,7 +111,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def test_http_requests(self):
-retries = 3
+self.skipTest("XXX This integration test fails non deterministically")
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry
@@ -120,17 +120,13 @@ class TestRunDirector(ConfigTestCase):
 assert 'factor' in entry
 assert 'headers_diff' in entry
 assert 'headers_match' in entry
-while retries > 0:
-try:
-yield self.run_helper('blocking/http_requests',
-['-u', 'http://torproject.org/'],
-verify_function)
-break
-except:
-retries -= 1
+yield self.run_helper('blocking/http_requests',
+['-u', 'http://torproject.org/'],
+verify_function)
 
 @defer.inlineCallbacks
 def test_http_requests_with_file(self):
+self.skipTest("XXX This integration test fails non deterministically")
 def verify_function(entry):
 assert 'body_length_match' in entry
 assert 'body_proportion' in entry



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Revert "Revert "ui/web/server.py: pass binary, not unicode cookies""

[art]

commit 897041c08eb6a54e8b181781718412d1500e9800
Author: Simone Basso 
Date:   Mon Sep 19 12:12:42 2016 +0200

Revert "Revert "ui/web/server.py: pass binary, not unicode cookies""

This reverts commit baee35fe51e70547e6128885ee0a97e1937d95d0 because
apparently the error was on a unrelated code path.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index eec82d3..385bf18 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(u'XSRF-TOKEN')
+token_cookie = request.getCookie(b'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(u'XSRF-TOKEN',
+request.addCookie(b'XSRF-TOKEN',
   instance._xsrf_token,
-  path=u'/')
+  path=b'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = ''.join([rng.choice(token_space)
+self._xsrf_token = b''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix cache updating logic.

[art]

commit 79ed2a50afc489b925bd6eaa78f1342d60877955
Author: Arturo Filastò 
Date:   Fri Sep 16 17:52:06 2016 +0200

Fix cache updating logic.

* Fix typo in comment
---
 ooni/deck/store.py| 11 ---
 ooni/ui/web/server.py |  4 ++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index bf1eb7f..f30b0d6 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -92,15 +92,17 @@ class InputStore(object):
 yield self.create(country_code)
 
 def _update_cache(self):
+new_cache = {}
 descs = self.path.child("descriptors")
 if not descs.exists():
-self._cache = {}
+self._cache = new_cache
 return
 
 for fn in descs.listdir():
 with descs.child(fn).open("r") as in_fh:
 input_desc = json.load(in_fh)
-self._cache[input_desc.pop("id")] = input_desc
+new_cache[input_desc.pop("id")] = input_desc
+self._cache = new_cache
 self._cache_stale = False
 return
 
@@ -172,6 +174,7 @@ class DeckStore(object):
 deck_enabled_path.remove()
 
 def _update_cache(self):
+new_cache = {}
 for deck_path in self.available_directory.listdir():
 if not deck_path.endswith('.yaml'):
 continue
@@ -179,7 +182,9 @@ class DeckStore(object):
 deck = NGDeck(
 deck_path=self.available_directory.child(deck_path).path
 )
-self._cache[deck_id] = deck
+new_cache[deck_id] = deck
+self._cache = new_cache
+self._cache_stale = False
 
 def get(self, deck_id):
 if self._cache_stale:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 48f21c7..cee42ce 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -144,8 +144,8 @@ class LongPoller(object):
 
 class WebUIAPI(object):
 app = Klein()
-# Maximum number in seconds after which to return a result even if not
-# change happenned.
+# Maximum number in seconds after which to return a result even if no
+# change happened.
 _long_polling_timeout = 5
 _reactor = reactor
 _enable_xsrf_protection = True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Revert "ui/web/server.py: pass binary, not unicode cookies"

[art]

commit baee35fe51e70547e6128885ee0a97e1937d95d0
Author: Simone Basso 
Date:   Mon Sep 19 11:37:20 2016 +0200

Revert "ui/web/server.py: pass binary, not unicode cookies"

This reverts commit 21548e9f0f53100336ef4f22d423f5f728a117e5 since
now Travis is failing for all versions of Twisted.
---
 ooni/ui/web/server.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 385bf18..eec82d3 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -56,13 +56,13 @@ def xsrf_protect(check=True):
 @wraps(f)
 def wrapper(instance, request, *a, **kw):
 should_check = check and instance._enable_xsrf_protection
-token_cookie = request.getCookie(b'XSRF-TOKEN')
+token_cookie = request.getCookie(u'XSRF-TOKEN')
 token_header = request.getHeader(b"X-XSRF-TOKEN")
 if (token_cookie != instance._xsrf_token and
 instance._enable_xsrf_protection):
-request.addCookie(b'XSRF-TOKEN',
+request.addCookie(u'XSRF-TOKEN',
   instance._xsrf_token,
-  path=b'/')
+  path=u'/')
 if should_check and token_cookie != token_header:
 raise WebUIError(404, "Invalid XSRF token")
 return f(instance, request, *a, **kw)
@@ -161,7 +161,7 @@ class WebUIAPI(object):
 # We use a double submit token to protect against XSRF
 rng = SystemRandom()
 token_space = string.letters+string.digits
-self._xsrf_token = b''.join([rng.choice(token_space)
+self._xsrf_token = ''.join([rng.choice(token_space)
 for _ in range(30)])
 
 self._director_started = False



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] In virtualenvs the /usr/share path is actually prefix/share/

[art]

commit 38775a3d39f598e925170f5f30792a44415a8a74
Author: Arturo Filastò 
Date:   Tue Sep 13 15:41:11 2016 +0200

In virtualenvs the /usr/share path is actually prefix/share/
---
 ooni/settings.py | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 409e81e..be0025d 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -168,7 +168,7 @@ if IS_VIRTUALENV:
 )
 USR_SHARE_PATH = os.path.join(
 _PREFIX,
-'usr', 'share', 'ooni'
+'share', 'ooni'
 )
 ETC_PATH = os.path.join(
 _PREFIX,
diff --git a/setup.py b/setup.py
index 753cf07..e6031c6 100644
--- a/setup.py
+++ b/setup.py
@@ -152,7 +152,7 @@ def mkdir_p(path):
 try:
 os.makedirs(path)
 except OSError as ose:
-if ose != errno.EEXIST:
+if ose.errno != errno.EEXIST:
 raise
 
 class OoniInstall(InstallCommand):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump up the default timeout for tor to 400 seconds

[art]

commit 3f25dad2ed4ede50798465eb284dbbc8f52e5ee4
Author: Arturo Filastò 
Date:   Tue Sep 13 17:42:33 2016 +0200

Bump up the default timeout for tor to 400 seconds
---
 ooni/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index be0025d..14f135f 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -86,7 +86,7 @@ tor:
 #
 # This is the timeout after which we consider to to not have
 # bootstrapped properly.
-#timeout: 200
+#timeout: 400
 #torrc:
 #HTTPProxy: host:port
 #HTTPProxyAuthenticator: user:password
@@ -141,7 +141,7 @@ defaults = {
 "control_port": None,
 "bridges": None,
 "data_dir": None,
-"timeout": 200,
+"timeout": 400,
 "torrc": {}
 }
 }



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Move check for running process into utility function

[art]

commit 8c3a65edc75e6830f4e5c7a6c2e1afd55851de3e
Author: Arturo Filastò 
Date:   Thu Sep 15 16:22:12 2016 +0200

Move check for running process into utility function

* More fixes based on review by @bassosimone
---
 ooni/measurements.py| 18 +++---
 ooni/nettest.py |  2 +-
 ooni/scripts/ooniprobe_agent.py | 16 
 ooni/utils/__init__.py  | 15 +++
 4 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/ooni/measurements.py b/ooni/measurements.py
index 294b182..422fc55 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -1,9 +1,7 @@
-import os
 import json
-import signal
 
 from twisted.python.filepath import FilePath
-from ooni.utils import log
+from ooni.utils import log, is_process_running
 from ooni.utils.files import directory_usage
 from ooni.settings import config
 
@@ -75,14 +73,11 @@ def get_measurement(measurement_id, compute_size=False):
 stale = False
 if measurement.child("measurements.njson.progress").exists():
 completed = False
-# XXX this is done quite often around the code, probably should
-# be moved into some utility function.
 pid = measurement.child("running.pid").open("r").read()
 pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
+if is_process_running(pid):
 running = True
-except OSError:
+else:
 stale = True
 
 if measurement.child("keep").exists():
@@ -136,10 +131,3 @@ def list_measurements(compute_size=False):
 except:
 log.err("Failed to get metadata for measurement 
{0}".format(measurement_id))
 return measurements
-
-if __name__ == "__main__":
-import sys
-if len(sys.argv) != 3:
-print("Usage: {0} [input_file] [output_file]".format(sys.argv[0]))
-sys.exit(1)
-generate_summary(sys.argv[1], sys.argv[2])
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 4ea3329..d6784f4 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -242,7 +242,7 @@ class NetTestLoader(object):
 m = ONION_INPUT_REGEXP.match(filename)
 if m:
 raise e.InvalidInputFile("Input files hosted on hidden services "
- "are not longer supported")
+ "are no longer supported")
 else:
 input_file['filename'] = filename
 self.inputFiles.append(input_file)
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 3eb1d22..c455f9b 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -8,7 +8,7 @@ import signal
 from twisted.scripts import twistd
 from twisted.python import usage
 
-from ooni.utils import log
+from ooni.utils import log, is_process_running
 from ooni.settings import config
 from ooni.agent.agent import AgentService
 
@@ -102,18 +102,10 @@ def get_running_pidfile():
 continue
 pid = open(pidfile, "r").read()
 pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
+if is_process_running(pid):
 running_pidfile = pidfile
-break
-except OSError as ose:
-if ose.errno == errno.ESRCH:
-# Found pid, but isn't running
-continue
-elif ose.errno == errno.EPERM:
-# The process is owned by root. We assume it's running
-running_pidfile = pidfile
-break
+else:
+continue
 if running_pidfile is None:
 raise NotRunning
 return running_pidfile
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 247758f..a894daf 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -1,6 +1,8 @@
 import shutil
 import string
 import random
+import signal
+import errno
 import gzip
 import os
 
@@ -160,3 +162,16 @@ def gunzip(file_path):
 def get_ooni_root():
 script = os.path.join(__file__, '..')
 return os.path.dirname(os.path.realpath(script))
+
+def is_process_running(pid):
+try:
+os.kill(pid, signal.SIG_DFL)
+running = True
+except OSError as ose:
+if ose.errno == errno.EPERM:
+running = True
+elif ose.errno == errno.ESRCH:
+running = False
+else:
+raise
+return running



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The last runtime should be determined based on the time we started the task

[art]

commit 5c0fd074630b7fdd4e066c0b783242880d0838a3
Author: Arturo Filastò 
Date:   Mon Sep 12 14:53:32 2016 +0200

The last runtime should be determined based on the time we started the task
---
 ooni/agent/scheduler.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 4002369..a7713e0 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -93,10 +93,9 @@ class ScheduledTask(object):
 date_str = in_file.read()
 return datetime.strptime(date_str, self._time_format)
 
-def _update_last_run(self):
+def _update_last_run(self, last_run_time):
 with self._last_run.open('w') as out_file:
-current_time = datetime.utcnow()
-out_file.write(current_time.strftime(self._time_format))
+out_file.write(last_run_time.strftime(self._time_format))
 
 def task(self):
 raise NotImplementedError
@@ -117,8 +116,9 @@ class ScheduledTask(object):
 try:
 if self.last_run == CANARY_DATE:
 yield defer.maybeDeferred(self.first_run)
+last_run_time = datetime.utcnow()
 yield self.task()
-self._update_last_run()
+self._update_last_run(last_run_time)
 except:
 raise
 finally:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] ooniprobe_agent: fix behavior when no command is specified

[art]

commit 300660ff9873d45d78e54b4a9c09c7c659afe6f5
Author: Simone Basso 
Date:   Mon Sep 19 11:02:17 2016 +0200

ooniprobe_agent: fix behavior when no command is specified
---
 ooni/scripts/ooniprobe_agent.py | 21 +++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 4c97f5e..49694eb 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -12,6 +12,7 @@ from twisted.python import usage
 from ooni.utils import log, is_process_running
 from ooni.settings import config
 from ooni.agent.agent import AgentService
+from ooni import __version__
 
 
 class StartOoniprobeAgentPlugin:
@@ -38,6 +39,10 @@ class RunOptions(usage.Options):
 pass
 
 class AgentOptions(usage.Options):
+
+synopsis = """%s [options] command
+""" % (os.path.basename(sys.argv[0]),)
+
 subCommands = [
 ['start', None, StartOptions, "Start the ooniprobe-agent in the "
   "background"],
@@ -45,9 +50,17 @@ class AgentOptions(usage.Options):
 ['status', None, StatusOptions, "Show status of the ooniprobe-agent"],
 ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"]
 ]
+
 def postOptions(self):
 self.twistd_args = []
 
+def opt_version(self):
+"""
+Display the ooniprobe version and exit.
+"""
+print("ooniprobe-agent version:", __version__)
+sys.exit(0)
+
 def start_agent(options=None):
 config.set_paths()
 config.initialize_ooni_home()
@@ -188,8 +201,9 @@ def run():
 options = AgentOptions()
 options.parseOptions()
 
-if options.subCommand == "run":
-options.twistd_args += ("--nodaemon",)
+if options.subCommand == None:
+print(options)
+return
 
 if options.subCommand == "stop":
 return stop_agent()
@@ -197,6 +211,9 @@ def run():
 if options.subCommand == "status":
 return status_agent()
 
+if options.subCommand == "run":
+options.twistd_args += ("--nodaemon",)
+
 return start_agent(options)
 
 if __name__ == "__main__":



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump to release candidate 3

[art]

commit b6b03acf87c44e798480deefd86ce2bb5926b7c0
Author: Arturo Filastò 
Date:   Mon Sep 19 13:38:27 2016 +0200

Bump to release candidate 3
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 20bdef0..41f5c37 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-rc.2"
+__version__ = "2.0.0-rc.3"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Use is_process_running function in reporter

[art]

commit 379533a802272ad34bbd1353493b6d9bf8d8449b
Author: Arturo Filastò 
Date:   Fri Sep 16 11:55:57 2016 +0200

Use is_process_running function in reporter
---
 ooni/reporter.py | 11 +++
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index b149d49..951602f 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -17,7 +17,7 @@ from twisted.python.util import untilConcludes
 from twisted.internet import defer
 from twisted.internet.error import ConnectionRefusedError
 
-from ooni.utils import log
+from ooni.utils import log, is_process_running
 from ooni.tasks import Measurement
 try:
 from scapy.packet import Packet
@@ -458,9 +458,7 @@ class OONIBReportLog(object):
 if entry['completed'] is False:
 continue
 if entry['status'] in ('created',):
-try:
-os.kill(entry['pid'], 0)
-except OSError:
+if not is_process_running(entry['pid']):
 incomplete_reports.append(
 (entry['measurements_path'], entry)
 )
@@ -476,13 +474,10 @@ class OONIBReportLog(object):
 all_entries = yield self.get_report_log_entries()
 for entry in all_entries[:]:
 if entry['status'] in ('created',):
-try:
-os.kill(entry['pid'], 0)
+if is_process_running(entry['pid']):
 in_progress_reports.append(
 (entry['measurements_path'], entry)
 )
-except OSError:
-pass
 defer.returnValue(in_progress_reports)
 
 @defer.inlineCallbacks



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add keyword arguments to ProbeIP.lookup method

[art]

commit 60fd4ec19818f77f68824c59db17739f86c87914
Author: Arturo Filastò 
Date:   Fri Sep 16 11:55:07 2016 +0200

Add keyword arguments to ProbeIP.lookup method
---
 ooni/geoip.py   | 17 ++---
 ooni/scripts/oonideckgen.py |  3 +--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/ooni/geoip.py b/ooni/geoip.py
index 40fad25..5e6b275 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -192,20 +192,23 @@ class ProbeIP(object):
 self._reset_state()
 return failure
 
-def resolveGeodata(self):
+def resolveGeodata(self,
+   include_ip=None,
+   include_asn=None,
+   include_country=None):
 from ooni.settings import config
 
 self.geodata = ip_to_location(self.address)
 self.geodata['ip'] = self.address
-if not config.privacy.includeasn:
+if not config.privacy.includeasn and include_asn is not True:
 self.geodata['asn'] = 'AS0'
-if not config.privacy.includecountry:
+if not config.privacy.includecountry and include_country is not True:
 self.geodata['countrycode'] = 'ZZ'
-if not config.privacy.includeip:
+if not config.privacy.includeip and include_ip is not True:
 self.geodata['ip'] = '127.0.0.1'
 
 @defer.inlineCallbacks
-def lookup(self):
+def lookup(self, include_ip=None, include_asn=None, include_country=None):
 if self._state == IN_PROGRESS:
 yield self._looking_up
 elif self._last_lookup < time.time() - self._expire_in:
@@ -218,7 +221,7 @@ class ProbeIP(object):
 try:
 yield self.askTor()
 log.msg("Found your IP via Tor")
-self.resolveGeodata()
+self.resolveGeodata(include_ip, include_asn, include_country)
 self._looking_up.callback(self.address)
 defer.returnValue(self.address)
 except errors.TorStateNotFound:
@@ -229,7 +232,7 @@ class ProbeIP(object):
 try:
 yield self.askGeoIPService()
 log.msg("Found your IP via a GeoIP service")
-self.resolveGeodata()
+self.resolveGeodata(include_ip, include_asn, include_country)
 self._looking_up.callback(self.address)
 defer.returnValue(self.address)
 except Exception as exc:
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 6c2882c..4ad0f59 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -84,8 +84,7 @@ def generate_deck(options):
 
 @defer.inlineCallbacks
 def get_user_country_code():
-config.privacy.includecountry = True
-yield probe_ip.lookup()
+yield probe_ip.lookup(include_country=True)
 defer.returnValue(probe_ip.geodata['countrycode'])
 
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add note about locking logic for run() of tasks

[art]

commit 6d13ade8bf4faae42e18c4e2c79dc695891d75c4
Author: Arturo Filastò 
Date:   Thu Sep 15 19:39:17 2016 +0200

Add note about locking logic for run() of tasks
---
 ooni/agent/scheduler.py | 9 +
 1 file changed, 9 insertions(+)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index c86e3b6..e8fef53 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -398,6 +398,15 @@ class SchedulerService(service.MultiService):
 """
 This function is called every self.interval seconds to check
 which periodic tasks should be run.
+
+Note: the task will wait on the lock if there is already a task of
+that type running. This means that if a task is very long running
+there can potentially be a pretty large backlog of accumulated
+periodic tasks waiting to know if they should run.
+XXX
+We may want to do something like not wait on the lock if there is
+already a queue that is larger than a certain amount or something
+smarter if still starts to become a memory usage concern.
 """
 for task in self._scheduled_tasks:
 log.debug("Running task {0}".format(task.identifier))



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update Changelog and version for 2.0.0-rc.2

[art]

commit 1fd8ad6526f4d49ad8bcd387a0d912efff568582
Author: Arturo Filastò 
Date:   Tue Sep 13 12:38:24 2016 +0200

Update Changelog and version for 2.0.0-rc.2
---
 ChangeLog.rst| 18 ++
 ooni/__init__.py |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 678983c..3c0ecd8 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,24 @@
 Changelog
 =
 
+v2.0.0-rc.2 (Tue, 13 Sep 2016)
+--
+
+This is a release candidate for a major ooniprobe release.
+
+It includes a new web user interface and a system daemon for running ooniprobe
+tests.
+
+Feature list:
+
+* System daemon for running tests periodically 
(https://github.com/TheTorProject/ooni-probe/issues/576)
+
+* Web user interface for viewing measurement results 
(https://github.com/TheTorProject/ooni-probe/issues/575)
+
+* New deck format (https://github.com/TheTorProject/ooni-probe/issues/571)
+
+* Local reports are written in JSON 
(https://github.com/TheTorProject/ooni-probe/issues/557)
+
 v1.6.1 (Tue, 26 Jul 2016)
 -
 
diff --git a/ooni/__init__.py b/ooni/__init__.py
index de2157c..20bdef0 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-rc.1"
+__version__ = "2.0.0-rc.2"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Make changes to the updater based on feedback by @bassosimone

[art]

commit 45993baf6bf0ae5377dcc9f99d9bf1c19a050b0b
Author: Arturo Filastò 
Date:   Thu Sep 15 12:41:42 2016 +0200

Make changes to the updater based on feedback by @bassosimone
---
 MANIFEST.in   |   2 +-
 data/lepidopter-update.py | 385 ++
 data/updater.py   | 357 --
 setup.py  |   2 +-
 4 files changed, 387 insertions(+), 359 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 0528d4b..258459e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,7 +7,7 @@ include data/oonireport.1
 include data/ooniresources.1
 include data/ooniprobe.conf.sample
 
-include data/updater.py
+include data/lepidopter-update.py
 
 include ooni/settings.ini
 include ooni/ui/consent-form.md
diff --git a/data/lepidopter-update.py b/data/lepidopter-update.py
new file mode 100755
index 000..bdd1cb5
--- /dev/null
+++ b/data/lepidopter-update.py
@@ -0,0 +1,385 @@
+#!/usr/bin/env python2
+"""
+This is the auto-updater script for lepidopter.
+
+It must be run from root and it takes care of downloading the most recent
+updates and doing all the operations needed to perform the update.
+
+To run it expects systemd to be configured.
+
+This script includes a self-installer which can be run via:
+
+python updater.py install
+
+It then expects to be run as a systemd service with:
+
+python updater.py update --watch
+"""
+
+from __future__ import print_function
+
+import os
+import re
+import imp # XPY3 this is deprecated in python3
+import sys
+import time
+import errno
+import shutil
+import getpass
+import logging
+import tempfile
+import argparse
+
+from subprocess import check_output, check_call, CalledProcessError
+
+# The version number of the updater
+__version__ = "1.0.0"
+
+LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
+# UPDATE_BASE_URL/latest/version must return an integer containing the latest 
version number
+# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION
+# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for 
update.py
+UPDATE_BASE_URL = 
"https://github.com/OpenObservatory/lepidopter-update/releases/download/;
+
+CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
+UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/"
+SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py"
+
+SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
+SYSTEMD_SCRIPT = """\
+[Unit]
+Description=lepidopter-update service
+
+[Service]
+Type=simple
+ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch
+TimeoutStartSec=300
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+""".format(SCRIPT_INSTALL_PATH)
+
+PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc"
+PUBLIC_KEY = """\
+-BEGIN PGP PUBLIC KEY BLOCK-
+Comment: GPGTools - https://gpgtools.org
+
+mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o
+9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4
+35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw
+520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA
+TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/
+7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x
+e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8
+6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI
+6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7
+ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG
+Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB
+tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0
+b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID
+AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d
+2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo
+CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+
+BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA
+i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R
+qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx
+bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ
+SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL
+f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l
+CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i
+S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66
+U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E
+XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf
+ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK
+sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l
+eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c
+qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS
+VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG

[tor-commits] [ooni-probe/master] Force creation of decks-available directory.

[art]

commit 28a441b1da39b8b8de28752dcfc0578352f6911c
Author: Arturo Filastò 
Date:   Tue Sep 13 15:28:02 2016 +0200

Force creation of decks-available directory.

Failing to do so will lead to the decks not being copied over and pip will
silently fail.
---
 setup.py | 19 +++
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index 60d08a5..753cf07 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,7 @@ Have fun!
 from __future__ import print_function
 
 import os
+import errno
 import tempfile
 from glob import glob
 
@@ -147,6 +148,13 @@ def install_lepidopter_update():
 check_call(["data/updater.py", "install"])
 
 
+def mkdir_p(path):
+try:
+os.makedirs(path)
+except OSError as ose:
+if ose != errno.EEXIST:
+raise
+
 class OoniInstall(InstallCommand):
 def gen_config(self, share_path):
 config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
@@ -183,14 +191,9 @@ class OoniInstall(InstallCommand):
 with open("ooni/settings.ini", "w+") as fp:
 settings.write(fp)
 
-try:
-os.makedirs(pj(var_path, 'ooni'))
-except OSError:
-pass
-try:
-os.makedirs(pj(share_path, 'ooni'))
-except OSError:
-pass
+mkdir_p(pj(var_path, 'ooni'))
+mkdir_p(pj(share_path, 'ooni'))
+mkdir_p(pj(share_path, 'ooni', 'decks-available'))
 
 def pre_install(self):
 prefix = os.path.abspath(self.prefix)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] When we talk about time ALWAYS use UTC

[art]

commit 46027a39a768efa6beabac9af62765970762f6bf
Author: Arturo Filastò 
Date:   Tue Sep 13 13:21:36 2016 +0200

When we talk about time ALWAYS use UTC
---
 ooni/agent/scheduler.py | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 0a6cc0f..d6fbc31 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -50,11 +50,13 @@ class FileSystemlockAndMutex(object):
 
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
-CANARY_DATE = datetime(1957, 8, 4)
+CANARY_DATE = datetime(1957, 8, 4, tzinfo=tz.tzutc())
+
 
 class DidNotRun(Exception):
 pass
 
+
 class ScheduledTask(object):
 _time_format = "%Y-%m-%dT%H:%M:%SZ"
 schedule = None
@@ -80,7 +82,7 @@ class ScheduledTask(object):
 
 @property
 def should_run(self):
-current_time = datetime.utcnow()
+current_time = datetime.utcnow().replace(tzinfo=tz.tzutc())
 next_cycle = croniter(self.schedule, self.last_run).get_next(datetime)
 if next_cycle <= current_time:
 return True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update updater script

[art]

commit 1d5d892181bf06f21b7c8df8566eb0906212382b
Author: Arturo Filastò 
Date:   Tue Sep 13 12:32:55 2016 +0200

Update updater script
---
 data/updater.py | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/data/updater.py b/data/updater.py
index 65983aa..f7225dd 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -198,9 +198,12 @@ def perform_update(version, skip_verification=False):
 
 try:
 logging.info("Running install script")
+if updater.__version__ != str(version):
+logging.error("There is a version mismatch in the updater file. 
This could be a sign of a replay attack.")
+raise UpdateFailed
 updater.run()
 except Exception:
-logging.error("Failed to run the version update script for version 
{0}".format(version))
+logging.exception("Failed to run the version update script for version 
{0}".format(version))
 raise UpdateFailed
 
 current_version_dir = os.path.dirname(CURRENT_VERSION_PATH)
@@ -214,6 +217,8 @@ def perform_update(version, skip_verification=False):
 with open(CURRENT_VERSION_PATH, "w+") as out_file:
 out_file.write(str(version))
 
+logging.info("Updated to version {0}".format(version))
+
 def update_to_version(from_version, to_version, skip_verification=False):
 versions = range(from_version + 1, to_version + 1)
 for version in versions:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add some debugging logs to the refresh deck list

[art]

commit a3700b5eabd923886f0e6951784196bdc451caaa
Author: Arturo Filastò 
Date:   Tue Sep 13 17:20:34 2016 +0200

Add some debugging logs to the refresh deck list
---
 ooni/agent/scheduler.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index d6fbc31..2797fba 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -278,6 +278,7 @@ class RefreshDeckList(ScheduledTask):
 def task(self):
 self.scheduler.refresh_deck_list()
 
+
 class SendHeartBeat(ScheduledTask):
 """
 This task is used to send a heartbeat that the probe is still alive and
@@ -345,6 +346,7 @@ class SchedulerService(service.MultiService):
 
 # If we are not initialized we should not enable anything
 if not config.is_initialized():
+log.msg("We are not initialized skipping setup of decks")
 to_enable = []
 
 for scheduled_task in self._scheduled_tasks[:]:
@@ -355,13 +357,16 @@ class SchedulerService(service.MultiService):
 if info in to_enable:
 # If the task is already scheduled there is no need to
 # enable it.
+log.msg("The deck {0} is already scheduled".format(deck_id))
 to_enable.remove(info)
 else:
 # If one of the tasks that is scheduled is no longer in the
 # scheduled tasks. We should disable it.
+log.msg("The deck task {0} should be disabled".format(deck_id))
 self.unschedule(scheduled_task)
 
 for deck_id, schedule in to_enable:
+log.msg("Scheduling to run {0}".format(deck_id))
 self.schedule(RunDeck(self.director, deck_id, schedule))
 
 def _task_did_not_run(self, failure, task):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Address feedback from @bassosimone

[art]

commit 47b4587d22993343a26aef6eef3667fe959e005e
Author: Arturo Filastò 
Date:   Wed Sep 14 17:03:21 2016 +0200

Address feedback from @bassosimone
---
 Vagrantfile|  5 -
 data/ooniprobe.conf.sample |  2 +-
 ooni/agent/scheduler.py| 24 +++-
 ooni/deck/backend.py   |  4 ++--
 ooni/deck/deck.py  |  2 +-
 ooni/director.py   |  6 +-
 ooni/settings.py   |  2 +-
 7 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 8ff0bbd..71f7c7a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,9 +81,4 @@ Vagrant.configure("2") do |config|
 end
   end
 
-  config.vm.define "testing" do |testing|
-testing.vm.network "forwarded_port", guest: 8842, host: 8142
-testing.vm.synced_folder ".", "/data/ooni-probe"
-  end
-
 end
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index f4c0812..ee9a2c9 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -3,7 +3,7 @@
 # Keep in mind that indentation matters.
 
 basic:
-# Where OONIProbe should be writing it's log file
+# Where OONIProbe should be writing its log file
 logfile: /var/log/ooniprobe.log
 loglevel: WARNING
 privacy:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 2797fba..c86e3b6 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -24,9 +24,12 @@ from ooni.measurements import list_measurements
 class FileSystemlockAndMutex(object):
 """
 This is a lock that is both a mutex lock and also on filesystem.
-When you acquire it, it will first block on the mutex lock and then
-once that is released it will attempt to acquire the lock on the
-filesystem.
+When you acquire it, it will first acquire the mutex lock and then
+acquire the filesystem lock. The release order is inverted.
+
+This is to avoid concurrent usage within the same process. When using it
+concurrently the mutex lock will block before the filesystem lock is
+acquired.
 
 It's a way to support concurrent usage of the DeferredFilesystemLock
 without races.
@@ -291,7 +294,7 @@ class SendHeartBeat(ScheduledTask):
 # XXX implement this
 pass
 
-# Order mattters
+# Order matters
 SYSTEM_TASKS = [
 UpdateInputsAndResources
 ]
@@ -303,7 +306,10 @@ def run_system_tasks(no_input_store=False):
 
 if no_input_store:
 log.debug("Not updating the inputs")
-task_classes.remove(UpdateInputsAndResources)
+try:
+task_classes.remove(UpdateInputsAndResources)
+except ValueError:
+pass
 
 for task_class in task_classes:
 task = task_class()
@@ -337,7 +343,15 @@ class SchedulerService(service.MultiService):
 self._scheduled_tasks.remove(task)
 
 def refresh_deck_list(self):
+"""
+This checks if there are some decks that have been enabled and
+should be scheduled as periodic tasks to run on the next scheduler
+cycle and if some have been disabled and should not be run.
 
+It does so by listing the enabled decks and checking if the enabled
+ones are already scheduled or if some of the scheduled ones are not
+amongst the enabled decks.
+"""
 to_enable = []
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
diff --git a/ooni/deck/backend.py b/ooni/deck/backend.py
index b2df9bc..a39492c 100644
--- a/ooni/deck/backend.py
+++ b/ooni/deck/backend.py
@@ -16,6 +16,8 @@ def sort_addresses_by_priority(priority_address,
 'address': priority_address,
 'type': backend_type
 }
+# We prefer an onion collector to an https collector to a cloudfront
+# collector to a plaintext collector
 address_priority = ['onion', 'https', 'cloudfront', 'http']
 address_priority.remove(preferred_backend)
 address_priority.insert(0, preferred_backend)
@@ -73,8 +75,6 @@ def get_reachable_test_helper(test_helper_name, 
test_helper_address,
 @defer.inlineCallbacks
 def get_reachable_collector(collector_address, collector_alternate,
 preferred_backend):
-# We prefer onion collector to https collector to cloudfront
-# collectors to plaintext collectors
 for collector_settings in sort_addresses_by_priority(
 collector_address,
 collector_alternate,
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 3fc10ab..59946ab 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -25,7 +25,7 @@ def resolve_file_path(v, prepath=None):
 if v.startswith("$"):
 # This raises InputNotFound and we let it carry onto the caller
 return input_store.get(v[1:])["filepath"]
-elif prepath is not None and (not os.path.isabs(v)):
+if prepath is not None and (not os.path.isabs(v)):
 return FilePath(prepath).preauthChild(v).path
 return v
 

[tor-commits] [ooni-probe/master] Enable submission of reports when run through the web user interface

[art]

commit 2ab3219a6044b27c47dd310c1e6cecec20f006fd
Author: Arturo Filastò 
Date:   Tue Sep 13 18:57:03 2016 +0200

Enable submission of reports when run through the web user interface
---
 ooni/ui/web/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index aed9951..48f21c7 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -458,7 +458,7 @@ class WebUIAPI(object):
 ]
 }
 try:
-deck = NGDeck(no_collector=True)
+deck = NGDeck()
 deck.load(deck_data)
 self.run_deck(deck)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Remove outdated comment

[art]

commit 0e452b1c70607da8d389988bb72e85b6b229fe4b
Author: Arturo Filastò 
Date:   Wed Sep 14 17:11:04 2016 +0200

Remove outdated comment
---
 ooni/resources.py | 5 -
 1 file changed, 5 deletions(-)

diff --git a/ooni/resources.py b/ooni/resources.py
index f97616c..ac0f967 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -5,11 +5,6 @@ from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
-# WARNING: this script is being run as part of the post install procedure.
-# Be sure to not import either in this module or in the imported modules
-# dependencies other than twisted. If you end up including something that is
-# not twisted, then you will need to add it to the setup_requires in setup.py.
-
 from ooni.utils import log, gunzip, rename
 from ooni.settings import config
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bugfix in updater

[art]

commit 96c7b6103023f4968141eb707266bf2bfa167368
Author: Arturo Filastò 
Date:   Wed Sep 7 13:36:43 2016 +0200

Bugfix in updater
---
 data/updater.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index 10086f5..65983aa 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -30,8 +30,7 @@ Description=lepidopter-update service
 
 [Service]
 Type=simple
-Environment="UPDATER={0}"
-ExecStart=$UPDATER --log-file /var/log/ooni/lepidopter-update.log update 
--watch
+ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch
 TimeoutStartSec=300
 Restart=on-failure
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Check permissions on logfile as well

[art]

commit bfc0a51a0b63ceb1d07812f2a7edc6a63847ea90
Author: Arturo Filastò 
Date:   Tue Sep 6 17:09:18 2016 +0200

Check permissions on logfile as well

* Handle the case where the measurement directory doesn't exist yet
---
 ooni/measurements.py | 2 ++
 ooni/utils/log.py| 7 ---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ooni/measurements.py b/ooni/measurements.py
index fd722ee..294b182 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -128,6 +128,8 @@ def get_summary(measurement_id):
 def list_measurements(compute_size=False):
 measurements = []
 measurement_path = FilePath(config.measurements_directory)
+if not measurement_path.exists():
+return measurements
 for measurement_id in measurement_path.listdir():
 try:
 measurements.append(get_measurement(measurement_id, compute_size))
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 67079e0..93e6b31 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -133,9 +133,10 @@ class OONILogger(object):
 logfile = os.path.expanduser(config.basic.logfile)
 
 log_folder = os.path.dirname(logfile)
-if not os.access(log_folder, os.W_OK):
-# If we don't have permissions to write to the log_folder,
-# write to running dir.
+if (not os.access(log_folder, os.W_OK) or
+(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
+# If we don't have permissions to write to the log_folder or
+# logfile.
 log_folder = config.running_path
 logfile = os.path.join(log_folder, "ooniprobe.log")
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Don't override the report_id

[art]

commit 406c60567a16f6a795602f59c7ec5a1f3e5bcce2
Author: Arturo Filastò 
Date:   Tue Sep 13 17:16:10 2016 +0200

Don't override the report_id
---
 ooni/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index cfdce08..b149d49 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -608,7 +608,7 @@ class Report(object):
 if self.collector_client:
 self.oonib_reporter = OONIBReporter(self.test_details,
 self.collector_client)
-self.test_details['report_id'] = yield self.open_oonib_reporter()
+yield self.open_oonib_reporter()
 
 if not self.no_njson:
 self.njson_reporter = NJSONReporter(self.test_details,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improve sanity checks on tor_data_dir

[art]

commit a731364e988798ebeb12e52eff1b1c11e18cdb2c
Author: Arturo Filastò 
Date:   Wed Sep 7 16:49:19 2016 +0200

Improve sanity checks on tor_data_dir
---
 ooni/utils/onion.py | 40 
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 6e0d906..10f80a3 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -1,6 +1,8 @@
 import os
 import re
 import pwd
+import fcntl
+import errno
 import string
 import StringIO
 import subprocess
@@ -214,6 +216,25 @@ def get_client_transport(transport):
 
 raise UninstalledTransport
 
+def is_tor_data_dir_usable(tor_data_dir):
+"""
+Checks if the Tor data dir specified is usable. This means that
+ it is not being locked and we have permissions to write to it.
+"""
+if not os.path.exists(tor_data_dir):
+return True
+
+try:
+fcntl.flock(open(os.path.join(tor_data_dir, 'lock'), 'w'),
+fcntl.LOCK_EX | fcntl.LOCK_NB)
+return True
+except (IOError, OSError) as err:
+if err.errno == errno.EACCES:
+# Permission error
+return False
+elif err.errno == errno.EAGAIN:
+# File locked
+return False
 
 def get_tor_config():
 tor_config = TorConfig()
@@ -227,11 +248,22 @@ def get_tor_config():
 
 if config.tor.data_dir:
 data_dir = os.path.expanduser(config.tor.data_dir)
-
-if not os.path.exists(data_dir):
-log.debug("%s does not exist. Creating it." % data_dir)
+# We only use the Tor data dir specified in the config file if
+# 1. It is not locked (i.e. another process is using it)
+# 2. We have write permissions to it
+data_dir_usable = is_tor_data_dir_usable(data_dir)
+try:
 os.makedirs(data_dir)
-tor_config.DataDirectory = data_dir
+log.debug("%s does not exist. Creating it." % data_dir)
+except OSError as ose:
+if ose.errno == errno.EEXIST:
+pass
+elif ose.errno == errno.EACCESS:
+data_dir_usable = False
+else:
+raise
+if data_dir_usable:
+tor_config.DataDirectory = data_dir
 
 if config.tor.bridges:
 tor_config.UseBridges = 1



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The updater needs to be updated

[art]

commit 5f3e8e311013d76db1dc62b21f66f0f3ce371c44
Author: Arturo Filastò 
Date:   Tue Sep 13 13:00:11 2016 +0200

The updater needs to be updated
---
 data/updater.py   | 6 +++---
 ooni/ui/web/client/index.html | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index f7225dd..4cacf3d 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -133,7 +133,7 @@ class InvalidPublicKey(Exception):
 pass
 
 
-def verify_file(signature_path, signer_pk_path):
+def verify_file(signature_path, file_path, signer_pk_path):
 tmp_dir = tempfile.mkdtemp()
 tmp_key = os.path.join(tmp_dir, "signing-key.gpg")
 
@@ -148,7 +148,7 @@ def verify_file(signature_path, signer_pk_path):
 output = check_output(["gpg", "--batch", "--status-fd", "1",
"--no-default-keyring", "--keyring",
tmp_key, "--trust-model", "always",
-   "--verify", signature_path])
+   "--verify", signature_path, file_path])
 except CalledProcessError:
 raise InvalidSignature
 
@@ -188,7 +188,7 @@ def perform_update(version, skip_verification=False):
 
 if skip_verification is not True:
 try:
-verify_file(updater_sig_path, PUBLIC_KEY_PATH)
+verify_file(updater_sig_path, updater_path, PUBLIC_KEY_PATH)
 except InvalidSignature:
 logging.error("Found an invalid signature. Bailing")
 raise UpdateFailed
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ecb4cdb..357b6eb 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
 
   Loading...
 
-  
+  
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bugfixes to quota management

[art]

commit e6b6f023681bd4e04efa7eb896f2daf9291451ef
Author: Arturo Filastò 
Date:   Mon Sep 5 18:32:45 2016 +0200

Bugfixes to quota management
---
 ooni/agent/scheduler.py | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 3cbd29d..21dd1cd 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -176,13 +176,13 @@ class CheckMeasurementQuota(ScheduledTask):
 if config.basic.measurement_quota is None:
 return
 maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
-available_bytes = directory_usage(config.measurements_directory)
+used_bytes = directory_usage(config.measurements_directory)
 warning_path = os.path.join(config.running_path, 'quota_warning')
 
-if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
+if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when:
 log.warn("You are about to reach the maximum allowed quota. Be 
careful")
 with open(warning_path, "w") as out_file:
-out_file.write("{0} {1}".format(available_bytes,
+out_file.write("{0} {1}".format(used_bytes,
 maximum_bytes))
 else:
 try:
@@ -191,12 +191,12 @@ class CheckMeasurementQuota(ScheduledTask):
 if ose.errno != errno.ENOENT:
 raise
 
-if float(available_bytes) < float(maximum_bytes):
+if float(used_bytes) < float(maximum_bytes):
 # We are within the allow quota exit.
 return
 
 # We should begin to delete old reports
-amount_to_delete = float(maximum_bytes) - float(available_bytes)
+amount_to_delete = float(used_bytes) - float(maximum_bytes)
 amount_deleted = 0
 measurement_path = FilePath(config.measurements_directory)
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Implement retry of director startup with bounded exponential binary backoff

[art]

commit 4f80a3bb07df8d742cb79f8ca918734412159422
Author: Arturo Filastò 
Date:   Thu Sep 8 16:17:28 2016 +0200

Implement retry of director startup with bounded exponential binary backoff
---
 ooni/director.py  |  7 +++
 ooni/geoip.py |  8 +++-
 ooni/ui/web/server.py | 29 -
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index cb2bdf0..1807e65 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -203,11 +203,9 @@ class Director(object):
 yield probe_ip.lookup()
 self.notify(DirectorEvent("success", "Looked up probe IP"))
 
-self.notify(DirectorEvent("success",
-  "Running system tasks"))
+self.notify(DirectorEvent("success", "Running system tasks"))
 yield run_system_tasks(no_input_store=not create_input_store)
-self.notify(DirectorEvent("success",
-  "Ran system tasks"))
+self.notify(DirectorEvent("success", "Ran system tasks"))
 
 @defer.inlineCallbacks
 def start(self, start_tor=False, check_incoherences=True,
@@ -218,6 +216,7 @@ class Director(object):
 self._director_starting.callback(self._director_state)
 except Exception as exc:
 self._director_starting.errback(Failure(exc))
+raise
 
 @property
 def measurementSuccessRatio(self):
diff --git a/ooni/geoip.py b/ooni/geoip.py
index f271790..40fad25 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -181,12 +181,17 @@ class ProbeIP(object):
 self._state = INITIAL
 self._looking_up = defer.Deferred()
 self._looking_up.addCallback(self._looked_up)
+self._looking_up.addErrback(self._lookup_failed)
 
 def _looked_up(self, result):
 self._last_lookup = time.time()
 self._reset_state()
 return result
 
+def _lookup_failed(self, failure):
+self._reset_state()
+return failure
+
 def resolveGeodata(self):
 from ooni.settings import config
 
@@ -227,8 +232,9 @@ class ProbeIP(object):
 self.resolveGeodata()
 self._looking_up.callback(self.address)
 defer.returnValue(self.address)
-except Exception:
+except Exception as exc:
 log.msg("Unable to lookup the probe IP via GeoIPService")
+self._looking_up.errback(defer.failure.Failure(exc))
 raise
 
 @defer.inlineCallbacks
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 26bfd47..aed9951 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -4,6 +4,7 @@ import os
 import json
 import errno
 import string
+import random
 from functools import wraps
 from random import SystemRandom
 
@@ -150,6 +151,7 @@ class WebUIAPI(object):
 _enable_xsrf_protection = True
 
 def __init__(self, config, director, scheduler, _reactor=reactor):
+self._reactor = reactor
 self.director = director
 self.scheduler = scheduler
 
@@ -165,6 +167,12 @@ class WebUIAPI(object):
 self._director_started = False
 self._is_initialized = config.is_initialized()
 
+# We use exponential backoff to trigger retries of the startup of
+# the director.
+self._director_startup_retries = 0
+# Maximum delay should be 6 hours
+self._director_max_retry_delay = 6*60*60
+
 self.status_poller = LongPoller(
 self._long_polling_timeout, _reactor)
 self.director_event_poller = LongPoller(
@@ -179,9 +187,11 @@ class WebUIAPI(object):
 self.start_director()
 
 def start_director(self):
+log.debug("Starting director")
 d = self.director.start()
 
 d.addCallback(self.director_started)
+d.addErrback(self.director_startup_failed)
 d.addBoth(lambda _: self.status_poller.notify())
 
 @property
@@ -208,7 +218,24 @@ class WebUIAPI(object):
 log.debug("Handling event {0}".format(event.type))
 self.director_event_poller.notify(event)
 
+def director_startup_failed(self, failure):
+self._director_startup_retries += 1
+
+# We delay the startup using binary exponential backoff with an
+# upper bound.
+startup_delay = random.uniform(
+0, min(2**self._director_startup_retries,
+   self._director_max_retry_delay)
+)
+log.err("Failed to start the director, "
+"retrying in {0}s".format(startup_delay))
+self._reactor.callLater(
+startup_delay,
+self.start_director
+)
+
 def director_started(self, _):
+log.debug("Started director")
 self._director_started = True
 
 @app.handle_errors(NotFound)
@@ -435,7 +462,7 @@ class WebUIAPI(object):
 deck.load(deck_data)
 

[tor-commits] [ooni-probe/master] Add more unittests for the scheduler

[art]

commit 4865ade8dc39d924e6a14e00c9b5a9e741f50abc
Author: Arturo Filastò 
Date:   Mon Sep 12 14:24:41 2016 +0200

Add more unittests for the scheduler

* Make the logic for triggering scheduled deck tasks more robust
---
 ooni/agent/scheduler.py  | 49 ++--
 ooni/tests/test_scheduler.py | 28 -
 2 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 98b395b..4002369 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,6 +1,7 @@
 import os
 import errno
 
+from hashlib import md5
 from datetime import datetime
 
 from twisted.application import service
@@ -41,6 +42,10 @@ class FileSystemlockAndMutex(object):
 self._fs_lock.unlock()
 self._mutex.release()
 
+@property
+def locked(self):
+return self._mutex.locked or self._fs_lock.locked
+
 # We use this date to indicate that the scheduled task has never run.
 # Easter egg, try to see what is special about this date :)?
 CANARY_DATE = datetime(1957, 8, 4)
@@ -68,6 +73,9 @@ class ScheduledTask(object):
 FilePath(scheduler_directory).child(self.identifier + ".lock").path
 )
 
+def cancel(self):
+self._last_run_lock.release()
+
 @property
 def should_run(self):
 current_time = datetime.utcnow()
@@ -233,7 +241,8 @@ class RunDeck(ScheduledTask):
 def __init__(self, director, deck_id, schedule):
 self.deck_id = deck_id
 self.director = director
-identifier = 'run-deck-' + deck_id
+# We use as identifier also the schedule time
+identifier = 'run-deck-' + deck_id + '-' + md5(schedule).hexdigest()
 super(RunDeck, self).__init__(schedule, identifier)
 
 @defer.inlineCallbacks
@@ -316,21 +325,39 @@ class SchedulerService(service.MultiService):
 def schedule(self, task):
 self._scheduled_tasks.append(task)
 
-def refresh_deck_list(self):
-# Deletes all the RunDeck tasks and reschedules only the ones that
-# are enabled.
-for scheduled_task in self._scheduled_tasks[:]:
-if isinstance(scheduled_task, RunDeck):
-self._scheduled_tasks.remove(scheduled_task)
+def unschedule(self, task):
+# We first cancel the task so the run lock is deleted
+task.cancel()
+self._scheduled_tasks.remove(task)
 
-if not config.is_initialized():
-# Disable scheduling measurements if we are not initialized.
-return
+def refresh_deck_list(self):
 
+to_enable = []
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
 continue
-self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+to_enable.append((deck_id, deck.schedule))
+
+# If we are not initialized we should not enable anything
+if not config.is_initialized():
+to_enable = []
+
+for scheduled_task in self._scheduled_tasks[:]:
+if not isinstance(scheduled_task, RunDeck):
+continue
+
+info = (scheduled_task.deck_id, scheduled_task.schedule)
+if info in to_enable:
+# If the task is already scheduled there is no need to
+# enable it.
+to_enable.remove(info)
+else:
+# If one of the tasks that is scheduled is no longer in the
+# scheduled tasks. We should disable it.
+self.unschedule(scheduled_task)
+
+for deck_id, schedule in to_enable:
+self.schedule(RunDeck(self.director, deck_id, schedule))
 
 def _task_did_not_run(self, failure, task):
 failure.trap(DidNotRun)
diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py
index cb90a30..1350dde 100644
--- a/ooni/tests/test_scheduler.py
+++ b/ooni/tests/test_scheduler.py
@@ -1,11 +1,12 @@
 import os
 import shutil
+import random
 import tempfile
 
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from ooni.agent.scheduler import ScheduledTask, DidNotRun
+from ooni.agent.scheduler import ScheduledTask, DidNotRun, 
FileSystemlockAndMutex
 
 class TestScheduler(unittest.TestCase):
 def test_scheduled_task(self):
@@ -49,3 +50,28 @@ class TestScheduler(unittest.TestCase):
 
 self.assertEqual(dummy_st.should_run, False)
 shutil.rmtree(scheduler_directory)
+
+
+@defer.inlineCallbacks
+def test_filesystem_lock_and_mutex(self):
+lock_dir = tempfile.mkdtemp()
+lock_path = os.path.join(lock_dir, 'lock')
+
+lock = FileSystemlockAndMutex(lock_path)
+
+lock_count = 100
+unlock_count = 0
+dl = []
+for i in range(lock_count):
+dl.append(lock.acquire())
+if random.choice([0, 1]) == 0:
+

[tor-commits] [ooni-probe/master] Convert last runtime into UTC

[art]

commit 5d0f080b6d968172171c16cdd9a9ebd8cb887972
Author: Arturo Filastò 
Date:   Mon Sep 12 17:00:27 2016 +0200

Convert last runtime into UTC
---
 ooni/agent/scheduler.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a7713e0..0a6cc0f 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -16,6 +16,8 @@ from ooni.utils.files import human_size_to_bytes, 
directory_usage
 from ooni.deck.store import input_store, deck_store, DEFAULT_DECKS
 from ooni.settings import config
 from ooni.contrib import croniter
+from ooni.contrib.dateutil.tz import tz
+
 from ooni.geoip import probe_ip
 from ooni.measurements import list_measurements
 
@@ -91,7 +93,8 @@ class ScheduledTask(object):
 return CANARY_DATE
 with self._last_run.open('r') as in_file:
 date_str = in_file.read()
-return datetime.strptime(date_str, self._time_format)
+return datetime.strptime(date_str, self._time_format).replace(
+tzinfo=tz.tzutc())
 
 def _update_last_run(self, last_run_time):
 with self._last_run.open('w') as out_file:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Better permissions handling for logfile and folder

[art]

commit 73b17d70ad8ac535ddebe36ca1b1d4c6277781d1
Author: Arturo Filastò 
Date:   Fri Sep 2 15:17:24 2016 +0200

Better permissions handling for logfile and folder
---
 ooni/utils/log.py | 15 +++
 1 file changed, 15 insertions(+)

diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 1aba8dd..67079e0 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import errno
 import codecs
 import logging
 
@@ -132,6 +133,20 @@ class OONILogger(object):
 logfile = os.path.expanduser(config.basic.logfile)
 
 log_folder = os.path.dirname(logfile)
+if not os.access(log_folder, os.W_OK):
+# If we don't have permissions to write to the log_folder,
+# write to running dir.
+log_folder = config.running_path
+logfile = os.path.join(log_folder, "ooniprobe.log")
+
+try:
+os.makedirs(log_folder)
+except OSError as ose:
+if ose.errno == errno.EEXIST and os.path.isdir(log_folder):
+pass
+else:
+raise
+
 log_filename = os.path.basename(logfile)
 file_log_level = levels.get(config.basic.loglevel,
 levels['INFO'])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update the updater

[art]

commit 8fa6e1ef7d7e1d8d7140e6171a7debabeffab908
Author: Arturo Filastò 
Date:   Wed Sep 7 13:21:27 2016 +0200

Update the updater
---
 data/updater.py | 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index 0f423ab..10086f5 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -20,8 +20,8 @@ from subprocess import check_output, check_call, 
CalledProcessError
 UPDATE_BASE_URL = 
"https://github.com/OpenObservatory/lepidopter-update/releases/download/;
 
 CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
-UPDATER_PATH = "/opt/ooni/updater/versions/"
-SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py"
+UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/"
+SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py"
 
 SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
 SYSTEMD_SCRIPT = """\
@@ -30,14 +30,16 @@ Description=lepidopter-update service
 
 [Service]
 Type=simple
-ExecStart={0} --log-file /var/log/lepidopter-update.log update --watch
+Environment="UPDATER={0}"
+ExecStart=$UPDATER --log-file /var/log/ooni/lepidopter-update.log update 
--watch
 TimeoutStartSec=300
 Restart=on-failure
+
 [Install]
 WantedBy=multi-user.target
 """.format(SCRIPT_INSTALL_PATH)
 
-PUBLIC_KEY_PATH = "/opt/ooni/updater/public.asc"
+PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc"
 PUBLIC_KEY = """\
 -BEGIN PGP PUBLIC KEY BLOCK-
 Comment: GPGTools - https://gpgtools.org
@@ -306,8 +308,8 @@ def install(args):
 with open(SYSTEMD_SCRIPT_PATH, "w") as out_file:
 out_file.write(SYSTEMD_SCRIPT)
 
-check_call(["systemctl", "enable", "lepidopter-updater"])
-check_call(["systemctl", "start", "lepidopter-updater"])
+check_call(["systemctl", "enable", "lepidopter-update"])
+check_call(["systemctl", "start", "lepidopter-update"])
 
 class InvalidLogLevel(Exception):
 pass



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Configure default decks when the scheduler is run for the first time.

[art]

commit 4e777252050d8b0266ce726ee8e43abfb6eaeb21
Author: Arturo Filastò 
Date:   Fri Sep 2 17:36:14 2016 +0200

Configure default decks when the scheduler is run for the first time.

* Add hook for running an operation when the task is run for the first time.
---
 ooni/agent/scheduler.py | 28 ++--
 ooni/deck/store.py  |  3 +++
 ooni/director.py|  2 +-
 ooni/settings.py| 33 -
 4 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 54551f6..7c2212b 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -12,7 +12,7 @@ from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
 from ooni.utils.files import human_size_to_bytes, directory_usage
-from ooni.deck.store import input_store, deck_store
+from ooni.deck.store import input_store, deck_store, DEFAULT_DECKS
 from ooni.settings import config
 from ooni.contrib import croniter
 from ooni.geoip import probe_ip
@@ -41,6 +41,10 @@ class FileSystemlockAndMutex(object):
 self._fs_lock.unlock()
 self._mutex.release()
 
+# We use this date to indicate that the scheduled task has never run.
+# Easter egg, try to see what is special about this date :)?
+CANARY_DATE = datetime(1957, 8, 4)
+
 class DidNotRun(Exception):
 pass
 
@@ -76,7 +80,7 @@ class ScheduledTask(object):
 def last_run(self):
 self._last_run.restat(False)
 if not self._last_run.exists():
-return datetime.fromtimestamp(0)
+return CANARY_DATE
 with self._last_run.open('r') as in_file:
 date_str = in_file.read()
 return datetime.strptime(date_str, self._time_format)
@@ -89,6 +93,13 @@ class ScheduledTask(object):
 def task(self):
 raise NotImplemented
 
+def first_run(self):
+"""
+This hook is called if it's the first time a particular scheduled
+operation is run.
+"""
+pass
+
 @defer.inlineCallbacks
 def run(self):
 yield self._last_run_lock.acquire()
@@ -96,6 +107,8 @@ class ScheduledTask(object):
 self._last_run_lock.release()
 raise DidNotRun
 try:
+if self.last_run == CANARY_DATE:
+yield defer.maybeDeferred(self.first_run)
 yield self.task()
 self._update_last_run()
 except:
@@ -240,6 +253,13 @@ class RefreshDeckList(ScheduledTask):
 self.scheduler = scheduler
 super(RefreshDeckList, self).__init__(schedule, identifier)
 
+def first_run(self):
+"""
+On first run we enable the default decks.
+"""
+for deck_id in DEFAULT_DECKS:
+deck_store.enable(deck_id)
+
 def task(self):
 self.scheduler.refresh_deck_list()
 
@@ -302,6 +322,10 @@ class SchedulerService(service.MultiService):
 if isinstance(scheduled_task, RunDeck):
 self._scheduled_tasks.remove(scheduled_task)
 
+if not config.is_initialized():
+# Disable scheduling measurements if we are not initialized.
+return
+
 for deck_id, deck in deck_store.list_enabled():
 if deck.schedule is None:
 continue
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index c938d56..bd852c7 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -11,6 +11,9 @@ from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
 from ooni.settings import config
 
+# These are the decks to be run by default.
+DEFAULT_DECKS = ['web-full']
+
 class InputNotFound(Exception):
 pass
 
diff --git a/ooni/director.py b/ooni/director.py
index ab31bf5..cb2bdf0 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -398,7 +398,7 @@ class Director(object):
 self._tor_starting.callback(self._tor_state)
 except Exception as exc:
 log.err("Failed to start tor")
-log.exc(exc)
+log.exception(exc)
 self._tor_starting.errback(Failure(exc))
 
 elif config.tor.control_port and config.tor_state is None:
diff --git a/ooni/settings.py b/ooni/settings.py
index 9f24f9b..a1c55fc 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -266,31 +266,6 @@ class OConfig(object):
 with open(initialized_path, 'w+'): pass
 
 @property
-def last_run_version(self):
-"""
-:return: Version identifying the last run version of ooniprobe.
-"""
-last_run_version_path = os.path.join(
-self.running_path, "last_run_version"
-)
-if not os.path.exists(last_run_version_path):
-return parse_version("0")
-with open(last_run_version_path) as in_file:
-last_run_version = in_file.read()
-return parse_version(last_run_version)

[tor-commits] [ooni-probe/master] Make settings parsing more robust

[art]

commit b334679f58c835b7a93b6f93662b7cf7aab80240
Author: Arturo Filastò 
Date:   Fri Sep 2 17:35:36 2016 +0200

Make settings parsing more robust

* Expand test case to cover more edge cases
---
 ooni/settings.py| 10 +++---
 ooni/tests/test_settings.py | 12 +---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 66bd212..9f24f9b 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -89,7 +89,7 @@ tor:
 # This is the timeout after which we consider to to not have
 # bootstrapped properly.
 #timeout: 200
-torrc:
+#torrc:
 #HTTPProxy: host:port
 #HTTPProxyAuthenticator: user:password
 #HTTPSProxy: host:port
@@ -213,14 +213,18 @@ def _load_config_files_with_defaults(config_files, 
defaults):
 continue
 with open(config_file_path) as in_file:
 c = yaml.safe_load(in_file)
-config_from_files.update(c)
+for category in c.keys():
+if c[category] is None:
+continue
+config_from_files[category] = config_from_files.get(category, {})
+config_from_files[category].update(c[category])
 
 for category in defaults.keys():
 configuration[category] = {}
 for k, v in defaults[category].items():
 try:
 configuration[category][k] = config_from_files[category][k]
-except KeyError:
+except (KeyError, TypeError):
 configuration[category][k] = defaults[category][k]
 return configuration
 
diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py
index 30d9e19..b88aa68 100644
--- a/ooni/tests/test_settings.py
+++ b/ooni/tests/test_settings.py
@@ -146,7 +146,8 @@ class TestSettings(ConfigTestCase):
 'key': 'value'
 },
 'cat2': {
-'key': 'value'
+'key': 'value',
+'key2': 'value2'
 },
 'cat3': {
 'key': 'value'
@@ -162,11 +163,15 @@ class TestSettings(ConfigTestCase):
 },
 'invalid_category': {
 'ignored': 'ignored'
-}
+},
+'cat3': None
 }
 config_file_B = {
 'cat1': {
 'key': 'valueB'
+},
+'cat2': {
+'key2': 'value2B'
 }
 }
 temp_dir = tempfile.mkdtemp()
@@ -188,7 +193,8 @@ class TestSettings(ConfigTestCase):
 'key': 'valueB'
 },
 'cat2': {
-'key': 'valueA'
+'key': 'valueA',
+'key2': 'value2B'
 },
 'cat3': {
 'key': 'value'



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add ooniprobe systemd service

[art]

commit 8da2312d61c4c1a840d0878e19967629526cd731
Author: Arturo Filastò 
Date:   Tue Sep 6 11:24:31 2016 +0200

Add ooniprobe systemd service
---
 scripts/systemd/ooniprobe.service | 15 +++
 1 file changed, 15 insertions(+)

diff --git a/scripts/systemd/ooniprobe.service 
b/scripts/systemd/ooniprobe.service
new file mode 100644
index 000..4743fac
--- /dev/null
+++ b/scripts/systemd/ooniprobe.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=%n, network interference detection tool
+After=network.target nss-lookup.target
+
+[Service]
+Type=forking
+PIDFile=/var/lib/ooni/twistd.pid
+ExecStart=/usr/local/bin/ooniprobe-agent start
+ExecStop=/usr/local/bin/ooniprobe-agent stop
+TimeoutStartSec=300
+TimeoutStopSec=60
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Update the updater

[art]

commit 1046f0ec3692f57d80ee73488bcdabc56d6e315c
Author: Arturo Filastò 
Date:   Tue Sep 6 12:50:09 2016 +0200

Update the updater
---
 data/updater.py | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/data/updater.py b/data/updater.py
index dc2fb2a..0f423ab 100755
--- a/data/updater.py
+++ b/data/updater.py
@@ -23,10 +23,10 @@ CURRENT_VERSION_PATH = "/etc/lepidopter-update/version"
 UPDATER_PATH = "/opt/ooni/updater/versions/"
 SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py"
 
-SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-updater.service"
+SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service"
 SYSTEMD_SCRIPT = """\
 [Unit]
-Description=lepidopter-updater service
+Description=lepidopter-update service
 
 [Service]
 Type=simple
@@ -138,12 +138,13 @@ def verify_file(signature_path, signer_pk_path):
 
 try:
 try:
-check_call(["gpg", "--yes", "-o", tmp_key, "--dearmor", 
signer_pk_path])
+check_call(["gpg", "--batch", "--yes", "-o", tmp_key,
+"--dearmor", signer_pk_path])
 except CalledProcessError:
 raise InvalidPublicKey
 
 try:
-output = check_output(["gpg", "--status-fd", "1",
+output = check_output(["gpg", "--batch", "--status-fd", "1",
"--no-default-keyring", "--keyring",
tmp_key, "--trust-model", "always",
"--verify", signature_path])



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add some basic unittests for the ScheduledTasks

[art]

commit 5723658b20d834dfd02b04264bf3f9e4cd8c9f50
Author: Arturo Filastò 
Date:   Tue Sep 6 16:50:03 2016 +0200

Add some basic unittests for the ScheduledTasks
---
 ooni/agent/scheduler.py  |  6 +++---
 ooni/settings.py |  2 --
 ooni/tests/test_scheduler.py | 51 
 3 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 21dd1cd..98b395b 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -53,7 +53,8 @@ class ScheduledTask(object):
 schedule = None
 identifier = None
 
-def __init__(self, schedule=None, identifier=None):
+def __init__(self, schedule=None, identifier=None,
+ scheduler_directory=config.scheduler_directory):
 if schedule is not None:
 self.schedule = schedule
 if identifier is not None:
@@ -61,7 +62,6 @@ class ScheduledTask(object):
 
 assert self.identifier is not None, "self.identifier must be set"
 assert self.schedule is not None, "self.schedule must be set"
-scheduler_directory = config.scheduler_directory
 
 self._last_run = FilePath(scheduler_directory).child(self.identifier)
 self._last_run_lock = FileSystemlockAndMutex(
@@ -91,7 +91,7 @@ class ScheduledTask(object):
 out_file.write(current_time.strftime(self._time_format))
 
 def task(self):
-raise NotImplemented
+raise NotImplementedError
 
 def first_run(self):
 """
diff --git a/ooni/settings.py b/ooni/settings.py
index a1c55fc..409e81e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,7 +3,6 @@ import sys
 import yaml
 import errno
 import getpass
-from pkg_resources import parse_version
 from ConfigParser import SafeConfigParser
 
 from twisted.internet import defer, reactor
@@ -11,7 +10,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from os.path import abspath, expanduser
 
-from ooni import __version__ as ooniprobe_version
 from ooni.utils import Storage, log, get_ooni_root
 
 CONFIG_FILE_TEMPLATE = """\
diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py
new file mode 100644
index 000..cb90a30
--- /dev/null
+++ b/ooni/tests/test_scheduler.py
@@ -0,0 +1,51 @@
+import os
+import shutil
+import tempfile
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from ooni.agent.scheduler import ScheduledTask, DidNotRun
+
+class TestScheduler(unittest.TestCase):
+def test_scheduled_task(self):
+schedule = "@daily"
+identifier = "dummy"
+scheduler_directory = tempfile.mkdtemp()
+scheduled_task = ScheduledTask(schedule=schedule,
+   identifier=identifier,
+   scheduler_directory=scheduler_directory
+)
+self.assertEqual(scheduled_task.should_run, True)
+self.assertFailure(scheduled_task.run(), NotImplementedError)
+self.assertEqual(scheduled_task.should_run, True)
+shutil.rmtree(scheduler_directory)
+
+@defer.inlineCallbacks
+def test_call_twice_scheduled_task(self):
+"""
+If we call the scheduled task twice in a row the second time it will 
not run.
+Tests for possible race conditions.
+"""
+scheduler_directory = tempfile.mkdtemp()
+spam_path = os.path.join(scheduler_directory, 'spam.txt')
+class DummyST(ScheduledTask):
+def task(self):
+with open(spam_path, 'w') as out_file:
+out_file.write("I ran\n")
+
+schedule = "@daily"
+identifier = "dummy"
+dummy_st = DummyST(schedule=schedule,
+   identifier=identifier,
+   scheduler_directory=scheduler_directory
+)
+
+dummy_st.run()
+yield self.assertFailure(dummy_st.run(), DidNotRun)
+
+with open(spam_path) as in_file:
+self.assertEqual(len(in_file.readlines()), 1)
+
+self.assertEqual(dummy_st.should_run, False)
+shutil.rmtree(scheduler_directory)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add to setup.py support for installing the updater

[art]

commit 21cf4b8525e911e89ee5c567ae5d375f88fda767
Author: Arturo Filastò 
Date:   Mon Sep 5 16:36:13 2016 +0200

Add to setup.py support for installing the updater
---
 MANIFEST.in |   5 +-
 data/configs/lepidopter-ooniprobe.conf  |  75 ---
 data/configs/lepidopter-oonireport.conf |  69 ---
 data/updater.py | 350 
 setup.py|  26 ++-
 5 files changed, 375 insertions(+), 150 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 60d2ef9..0528d4b 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,7 +6,8 @@ include data/ooniprobe.1
 include data/oonireport.1
 include data/ooniresources.1
 include data/ooniprobe.conf.sample
-include data/configs/lepidopter-ooniprobe.conf
-include data/configs/lepidopter-oonireport.conf
+
+include data/updater.py
+
 include ooni/settings.ini
 include ooni/ui/consent-form.md
diff --git a/data/configs/lepidopter-ooniprobe.conf 
b/data/configs/lepidopter-ooniprobe.conf
deleted file mode 100644
index 5c8ba40..000
--- a/data/configs/lepidopter-ooniprobe.conf
+++ /dev/null
@@ -1,75 +0,0 @@
-# This is the configuration file for OONIProbe
-# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
-# Keep in mind that indentation matters.
-
-basic:
-# Where OONIProbe should be writing it's log file
-logfile: /var/log/ooni/ooniprobe.log
-# in the future we will support loglevels
-loglevel: WARNING
-privacy:
-# Should we include the IP address of the probe in the report?
-includeip: false
-# Should we include the ASN of the probe in the report?
-includeasn: true
-# Should we include the country as reported by GeoIP in the report?
-includecountry: true
-# Should we include the city as reported by GeoIP in the report?
-includecity: false
-# Should we collect a full packet capture on the client?
-includepcap: false
-reports:
-# Should we place a unique ID inside of every report
-unique_id: true
-# This is a prefix for each packet capture file (.pcap) per test:
-pcap: null
-collector: null
-advanced:
-geoip_data_dir: /usr/share/GeoIP
-debug: false
-# enable if auto detection fails
-#tor_binary: /usr/sbin/tor
-#obfsproxy_binary: /usr/bin/obfsproxy 
-# For auto detection
-interface: auto
-# Of specify a specific interface
-#interface: wlan0
-# If you do not specify start_tor, you will have to have Tor running and
-# explicitly set the control port and SOCKS port
-start_tor: true
-# After how many seconds we should give up on a particular measurement
-measurement_timeout: 120
-# After how many retries we should give up on a measurement
-measurement_retries: 2
-# How many measurements to perform concurrently
-measurement_concurrency: 4
-# After how may seconds we should give up reporting
-reporting_timeout: 360
-# After how many retries to give up on reporting
-reporting_retries: 5
-# How many reports to perform concurrently
-reporting_concurrency: 7
-oonid_api_port: 8042
-report_log_file: null
-inputs_dir: null
-decks_dir: null
-tor:
-#socks_port: 8801
-#control_port: 8802
-# Specify the absolute path to the Tor bridges to use for testing
-#bridges: bridges.list
-# Specify path of the tor datadirectory.
-# This should be set to something to avoid having Tor download each time
-# the descriptors and consensus data.
-data_dir: /opt/ooni/tor_data_dir
-torrc:
-#HTTPProxy: host:port
-#HTTPProxyAuthenticator: user:password
-#HTTPSProxy: host:port
-#HTTPSProxyAuthenticator: user:password
-# Uncomment following 5 lines to connect via meek pluggable transport in 
tor
-#UseBridges: 1
-#Bridge:
-#- "meek_lite 0.0.2.0:1 url=https://d2zfqthxsdq309.cloudfront.net/ 
front=a0.awsstatic.com"
-#- "meek_lite 0.0.2.0:2 url=https://az786092.vo.msecnd.net/ 
front=ajax.aspnetcdn.com"
-#ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy"
diff --git a/data/configs/lepidopter-oonireport.conf 
b/data/configs/lepidopter-oonireport.conf
deleted file mode 100644
index 8ea3ccc..000
--- a/data/configs/lepidopter-oonireport.conf
+++ /dev/null
@@ -1,69 +0,0 @@
-# This is the configuration file for OONIProbe
-# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
-# Keep in mind that indentation matters.
-
-basic:
-# Where OONIProbe should be writing it's log file
-logfile: /var/log/ooni/oonireport.log
-# in the future we will support loglevels
-loglevel: WARNING
-privacy:
-# Should we include the IP address of the probe in the report?
-includeip: false
-# Should we include the ASN of the probe in the report?
-includeasn: true
-# Should we include the country as reported by GeoIP in the report?
-includecountry: true
-# Should 

[tor-commits] [ooni-probe/master] Fix typo in data deletion schedule

[art]

commit b41bc4b5c6e9616fd4a0ea50a770d862984379e2
Author: Arturo Filastò 
Date:   Mon Sep 5 15:54:40 2016 +0200

Fix typo in data deletion schedule
---
 ooni/agent/scheduler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 7c2212b..3cbd29d 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -182,7 +182,8 @@ class CheckMeasurementQuota(ScheduledTask):
 if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
 log.warn("You are about to reach the maximum allowed quota. Be 
careful")
 with open(warning_path, "w") as out_file:
-out_file.write("{0} {1}".split(available_bytes, maximum_bytes))
+out_file.write("{0} {1}".format(available_bytes,
+maximum_bytes))
 else:
 try:
 os.remove(warning_path)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Move lepidopter update install process to the install script

[art]

commit 1165aeee44567e604f24291a716ef2fb6202e38f
Author: Arturo Filastò 
Date:   Mon Sep 5 16:48:56 2016 +0200

Move lepidopter update install process to the install script
---
 setup.py | 17 +++--
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/setup.py b/setup.py
index c465789..60d08a5 100644
--- a/setup.py
+++ b/setup.py
@@ -143,7 +143,7 @@ def is_updater_installed():
 return os.path.exists('/etc/lepidopter-update/version')
 
 
-def install_updater():
+def install_lepidopter_update():
 check_call(["data/updater.py", "install"])
 
 
@@ -199,17 +199,14 @@ class OoniInstall(InstallCommand):
 def run(self):
 self.pre_install()
 self.do_egg_install()
-
+if is_lepidopter() and not is_updater_installed():
+print("Lepidopter now requires that ooniprobe is installed via the 
"
+ "updater")
+print("Let me install the auto-updater for you and we shall use 
that "
+  "for updates in the future.")
+install_lepidopter_update()
 
 def setup_package():
-if is_lepidopter() and not is_updater_installed():
-print("Lepidopter now requires that ooniprobe is installed via the "
-  "updater")
-print("Let me install the auto-updater for you and we shall use that "
-  "for updates in the future.")
-install_updater()
-return
-
 setup_requires = []
 install_requires = []
 dependency_links = []



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add simple unittest for api_status

[art]

commit 417e64893fa21a9ca1b9e66efa812d784a128731
Author: Arturo Filastò 
Date:   Tue Sep 6 16:08:27 2016 +0200

Add simple unittest for api_status
---
 .coveragerc   |  2 +-
 ooni/tests/test_wui_server.py | 32 
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/.coveragerc b/.coveragerc
index 34ed07c..4f5cfba 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,4 +1,4 @@
 [run]
 source = ooni
 [report]
-omit = ooni/nettests/*, ooni/api/*, ooni/kit/*
+omit = ooni/nettests/*, ooni/api/*, ooni/kit/*, ooni/contrib/*
diff --git a/ooni/tests/test_wui_server.py b/ooni/tests/test_wui_server.py
new file mode 100644
index 000..ebaaacd
--- /dev/null
+++ b/ooni/tests/test_wui_server.py
@@ -0,0 +1,32 @@
+from __future__ import print_function
+
+import json
+
+from .bases import ConfigTestCase
+from ooni.ui.web.server import WebUIAPI
+
+from mock import patch, Mock
+
+class TestWebUIServer(ConfigTestCase):
+def setUp(self):
+super(TestWebUIServer, self).setUp()
+
+director = Mock()
+scheduler = Mock()
+
+self.wui_api = WebUIAPI(self.config, director, scheduler)
+
+def tearDown(self):
+super(TestWebUIServer, self).tearDown()
+self.wui_api.status_poller.stop()
+self.wui_api.director_event_poller.stop()
+
+def test_api_status(self):
+request = Mock()
+resp = self.wui_api.api_status(request)
+j = json.loads(resp)
+expected_keys = ["software_version", "software_name", "asn",
+ "country_code", "director_started", "initialized",
+ "quota_warning"]
+for key in expected_keys:
+self.assertTrue(key in j)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bump version to rc.1

[art]

commit 7fe1bc1be0f657b8c2492408288067f2d4fd1d62
Author: Arturo Filastò 
Date:   Tue Sep 6 16:19:02 2016 +0200

Bump version to rc.1
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index d4b6f95..de2157c 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.3"
+__version__ = "2.0.0-rc.1"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for parsing multiple config files

[art]

commit 31bcae623b8a621aa941f284e67438168243c44a
Author: Arturo Filastò 
Date:   Fri Sep 2 15:00:48 2016 +0200

Add support for parsing multiple config files

* Configuration files are parsed in a certain order and they override the
  default behavior.
---
 Vagrantfile |   5 +
 ooni/__init__.py|   2 +-
 ooni/scripts/ooniprobe_agent.py |   1 +
 ooni/settings.py| 125 ++-
 ooni/tests/test_settings.py |  61 ++
 ooni/ui/cli.py  |   4 +-
 ooni/ui/web/client/index.html   |   2 +-
 setup.py| 262 ++--
 8 files changed, 285 insertions(+), 177 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 71f7c7a..8ff0bbd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,4 +81,9 @@ Vagrant.configure("2") do |config|
 end
   end
 
+  config.vm.define "testing" do |testing|
+testing.vm.network "forwarded_port", guest: 8842, host: 8142
+testing.vm.synced_folder ".", "/data/ooni-probe"
+  end
+
 end
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 9eecb67..d4b6f95 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.2"
+__version__ = "2.0.0-alpha.3"
 
 __all__ = [
 'agent',
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index f56f2df..3eb1d22 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -58,6 +58,7 @@ def start_agent(options=None):
 #  a null log observer
 twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
'--umask', '022']
+
 twistd_config = OoniprobeTwistdConfig()
 if options is not None:
 twistd_args.extend(options.twistd_args)
diff --git a/ooni/settings.py b/ooni/settings.py
index a0ef0a5..66bd212 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,6 +3,7 @@ import sys
 import yaml
 import errno
 import getpass
+from pkg_resources import parse_version
 from ConfigParser import SafeConfigParser
 
 from twisted.internet import defer, reactor
@@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from os.path import abspath, expanduser
 
+from ooni import __version__ as ooniprobe_version
 from ooni.utils import Storage, log, get_ooni_root
 
 CONFIG_FILE_TEMPLATE = """\
@@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\
 
 basic:
 # Where OONIProbe should be writing it's log file
-logfile: {logfile}
-loglevel: WARNING
+# logfile: {logfile}
+# loglevel: WARNING
 # The maximum amount of data to store on disk. Once the quota is reached,
 # we will start deleting older reports.
 # measurement_quota: 1G
@@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI):
 if _ETC_PATH is not None:
 ETC_PATH = _ETC_PATH
 
+
+def _load_config_files_with_defaults(config_files, defaults):
+"""
+This takes care of reading the config files in reverse order (the first
+item will have priority over the last element) and produce a
+configuration that includes ONLY the options inside of the defaults
+dictionary.
+
+:param config_files: a list of configuration file paths
+:param defaults: the default values for the configuration file
+:return: a configuration that is the result of reading the config files
+and joining it with the default options.
+"""
+config_from_files = {}
+configuration = {}
+for config_file_path in reversed(config_files):
+if not os.path.exists(config_file_path):
+continue
+with open(config_file_path) as in_file:
+c = yaml.safe_load(in_file)
+config_from_files.update(c)
+
+for category in defaults.keys():
+configuration[category] = {}
+for k, v in defaults[category].items():
+try:
+configuration[category][k] = config_from_files[category][k]
+except KeyError:
+configuration[category][k] = defaults[category][k]
+return configuration
+
 class OConfig(object):
 _custom_home = None
 
@@ -211,6 +244,10 @@ class OConfig(object):
 self.tor = Storage()
 self.privacy = Storage()
 
+# In here we store the configuration files ordered by priority.
+# First configuration file takes priority over the others.
+self.config_files = []
+
 self.set_paths()
 
 def is_initialized(self):
@@ -225,6 +262,31 @@ class OConfig(object):
 with open(initialized_path, 'w+'): pass
 
 @property
+def last_run_version(self):
+"""
+:return: Version identifying the last run version of ooniprobe.
+"""
+last_run_version_path = os.path.join(
+self.running_path, "last_run_version"
+)
+if not os.path.exists(last_run_version_path):
+  

[tor-commits] [ooni-probe/master] web-full.yaml: add missing word

[art]

commit eb7971a108201bed8f6735a14f7372ab9960
Author: Simone Basso 
Date:   Mon Aug 8 18:46:27 2016 +0200

web-full.yaml: add missing word
---
 data/decks/web-full.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
index 7812505..ccb470d 100644
--- a/data/decks/web-full.yaml
+++ b/data/decks/web-full.yaml
@@ -1,7 +1,7 @@
 ---
 name: Full Web test deck
 description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
-Request and the Web Connectivity test
+Request Line and the Web Connectivity test
 schedule: "@daily"
 tasks:
 - name: Runs the HTTP Header Field Manipulation test



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Generate the filename for the report late on to avoid conflicts in measurement IDs

[art]

commit 40b83a84265a24bd6777195b43a36edc8dc42640
Author: Arturo Filastò 
Date:   Tue Sep 6 22:05:15 2016 +0200

Generate the filename for the report late on to avoid conflicts in 
measurement IDs
---
 ooni/deck/deck.py | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index e4c35c4..3fc10ab 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,4 +1,6 @@
 import os
+import uuid
+import errno
 from copy import deepcopy
 from string import Template
 
@@ -90,7 +92,6 @@ class NGDeck(object):
 
 self._measurement_path = FilePath(config.measurements_directory)
 self._tasks = []
-self.task_ids = []
 
 if deck_path is not None:
 self.open(deck_path)
@@ -145,7 +146,6 @@ class NGDeck(object):
 self.bouncer.backend_type == "onion"):
 self.requires_tor = True
 self._tasks.append(deck_task)
-self.task_ids.append(deck_task.id)
 
 if self.metadata.get('no_collector', False):
 self.no_collector = True
@@ -217,7 +217,10 @@ class NGDeck(object):
 
 def _run_ooni_task(self, task, director):
 net_test_loader = task.ooni["net_test_loader"]
-test_details = task.ooni["test_details"]
+# XXX-REFACTOR we do this so late to avoid the collision between the
+#  same id and hence generating the same filename.
+test_details = net_test_loader.getTestDetails()
+task.id = generate_filename(test_details)
 
 measurement_id = None
 report_filename = task.output_path
@@ -228,9 +231,9 @@ class NGDeck(object):
 try:
 measurement_dir.createDirectory()
 except OSError as ose:
-# Ignore 'File Exists'
-if ose.errno != 17:
-raise
+if ose.errno == errno.EEXIST:
+raise Exception("Directory already exists, there is a "
+"collision")
 
 report_filename = 
measurement_dir.child("measurements.njson.progress").path
 pid_file = measurement_dir.child("running.pid")
@@ -388,10 +391,9 @@ class DeckTask(object):
 )
 file_path = resolve_file_path(filename, self.cwd)
 input_file['test_options'][input_file['key']] = file_path
-self.ooni['test_details'] = 
self.ooni['net_test_loader'].getTestDetails()
-self.id = generate_filename(self.ooni['test_details'])
 
 def setup(self):
+self.id = str(uuid.uuid4())
 return getattr(self, "_setup_"+self.type)()
 
 def _load(self, data):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Check for all the possible pid file locations when checking status of ooniprobe-agent

[art]

commit bb7bcdd18d6a94097cf4de057c752828cfdf16c6
Author: Arturo Filastò 
Date:   Thu Sep 1 17:29:23 2016 +0200

Check for all the possible pid file locations when checking status of 
ooniprobe-agent

* This fixes https://github.com/TheTorProject/ooni-probe/issues/597
---
 ooni/scripts/ooniprobe_agent.py | 41 -
 ooni/settings.py|  8 
 setup.py| 14 ++
 3 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 08fba89..ecfe7d8 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -67,6 +67,11 @@ def start_agent(options=None):
 twistd_config.loadedPlugins = {
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
+
+if status_agent() == 0:
+print("Stop ooniprobe-agent before attempting to start it")
+return 1
+
 print("Starting ooniprobe agent.")
 WEB_UI_URL = "http://{0}:{1}".format(
 config.advanced.webui_address, config.advanced.webui_port)
@@ -76,23 +81,25 @@ def start_agent(options=None):
 return 0
 
 def status_agent():
-pidfile = os.path.join(
-config.running_path,
-'twistd.pid'
-)
-if not os.path.exists(pidfile):
-print("ooniprobe-agent is NOT running")
-return  1
-pid = open(pidfile, "r").read()
-pid = int(pid)
-try:
-os.kill(pid, signal.SIG_DFL)
-except OSError, oserr:
-if oserr.errno == 3:
-print("ooniprobe-agent is NOT running")
-return 1
-print("ooniprobe-agent is running")
-return 0
+running = False
+for pidfile in [config.system_pid_path, config.user_pid_path]:
+if not os.path.exists(pidfile):
+# Didn't find the pid_file
+continue
+pid = open(pidfile, "r").read()
+pid = int(pid)
+try:
+os.kill(pid, signal.SIG_DFL)
+running = True
+except OSError, oserr:
+if oserr.errno == 3:
+# Found pid, but isn't running
+continue
+if running is True:
+print("ooniprobe-agent is running")
+return 0
+print("ooniprobe-agent is NOT running")
+return 1
 
 def stop_agent():
 # This function is borrowed from tahoe
diff --git a/ooni/settings.py b/ooni/settings.py
index c36f3f7..065252b 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -235,6 +235,14 @@ class OConfig(object):
 return self.ooni_home
 
 @property
+def user_pid_path(self):
+return os.path.join(self.ooni_home, "twistd.pid")
+
+@property
+def system_pid_path(self):
+return os.path.join(VAR_LIB_PATH, "twistd.pid")
+
+@property
 def data_directory_candidates(self):
 dirs = [
 self.ooni_home,
diff --git a/setup.py b/setup.py
index 7612b39..e5dd836 100644
--- a/setup.py
+++ b/setup.py
@@ -69,19 +69,17 @@ When you got them run:
 Using ooniprobe
 ---
 
-To generate a test deck for your country, cd to the directory where you want it
-and run:
+It is recommended that you start the ooniprobe-agent system daemon that will
+expose a localhost only Web UI and automatically run tests for you.
 
-.. code:: bash
-
-oonideckgen
+This can be done with:
 
+.. code:: bash
 
-To setup a daily cronjob run this:
+ooniprobe-agent start
 
-.. code:: bash
 
-(crontab -l 2>/dev/null; echo "@daily ooniprobe `oonideckgen | grep -e 
'^ooniprobe'`") | crontab -
+Then connect to the local web interface on http://127.0.0.1:8842/
 
 Have fun!
 """



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add deck configuration step to the setup wizard

[art]

commit 0c27e0602cfcfb172bc09d1ca2c35a612803f783
Author: Arturo Filastò 
Date:   Wed Aug 31 16:32:38 2016 +0200

Add deck configuration step to the setup wizard
---
 ooni/ui/web/server.py | 38 +-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 6f9ef69..2e89370 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -190,7 +190,8 @@ class WebUIAPI(object):
 def status(self):
 quota_warning = None
 try:
-with open(os.path.join(config.running_dir, "quota_warning")) as 
in_file:
+with open(os.path.join(config.running_path,
+   "quota_warning")) as in_file:
 quota_warning = in_file.read()
 except IOError as ioe:
 if ioe.errno != errno.ENOENT:
@@ -259,6 +260,21 @@ class WebUIAPI(object):
 d.addCallback(got_status_update)
 return d
 
+@app.route('/api/initialize', methods=["GET"])
+@xsrf_protect(check=False)
+@requires_false(attrs=['_is_initialized'])
+def api_initialize_get(self, request):
+available_decks = []
+for deck_id, deck in self.director.deck_store.list():
+available_decks.append({
+'name': deck.name,
+'description': deck.description,
+'schedule': deck.schedule,
+'enabled': self.director.deck_store.is_enabled(deck_id),
+'id': deck_id
+})
+return self.render_json({"available_decks": available_decks}, request)
+
 @app.route('/api/initialize', methods=["POST"])
 @xsrf_protect(check=True)
 @requires_false(attrs=['_is_initialized'])
@@ -278,6 +294,26 @@ class WebUIAPI(object):
 raise WebUIError(400, 'Missing required key {0}'.format(
 required_key))
 config.create_config_file(**options)
+try:
+deck_config = initial_configuration['deck_config']
+except KeyError:
+raise WebUIError(400, 'Missing enabled decks')
+
+for deck_id, enabled in deck_config.items():
+try:
+if enabled is True:
+self.director.deck_store.enable(deck_id)
+elif enabled is False:
+try:
+self.director.deck_store.disable(deck_id)
+except DeckNotFound:
+# We ignore these errors, because it could be that a 
deck
+# that is marked as disabled is already disabled
+pass
+except DeckNotFound:
+raise WebUIError(404, 'Deck not found')
+
+self.scheduler.refresh_deck_list()
 config.set_initialized()
 
 self._is_initialized = True



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add informed consent form to be used in the CLI version of ooniprobe

[art]

commit c23c47b4a314329b6bf9a778027b421801e55b82
Author: Arturo Filastò 
Date:   Thu Aug 4 19:47:15 2016 +0200

Add informed consent form to be used in the CLI version of ooniprobe
---
 ooni/ui/consent-form.md | 265 
 1 file changed, 265 insertions(+)

diff --git a/ooni/ui/consent-form.md b/ooni/ui/consent-form.md
new file mode 100644
index 000..08de2be
--- /dev/null
+++ b/ooni/ui/consent-form.md
@@ -0,0 +1,265 @@
+The [Open Observatory of Network Interference
+(OONI)](https://ooni.torproject.org/) is a free software project, under the 
[Tor
+Project](https://www.torproject.org/), which collects and processes network
+measurements with the aim of detecting network anomalies, such as censorship 
and
+traffic manipulation.
+
+Running OONI may be against the terms of service of your ISP or legally
+questionable in your country. By running OONI you will connect to web services
+which may be banned, and use web censorship circumvention methods such as Tor.
+The OONI project will publish data submitted by probes, possibly including your
+IP address or other identifying information. In addition, your use of OONI will
+be clear to anybody who has access to your computer, and to anybody who can
+monitor your internet connection (such as your employer, ISP or government).
+
+By running ooniprobe, you are participating as a volunteer in this project. 
This
+form includes information that you should be aware of and consent to *prior* to
+running ooniprobe.
+
+## OONI software tests
+
+The OONI project has developed multiple free software tests which are designed 
to:
+
+* Detect the blocking of websites
+
+* Detect systems responsible for censorship and traffic manipulation
+
+* Evaluate the reachability of [Tor bridges](https://bridges.torproject.org/),
+  proxies, VPNs, and sensitive domains
+
+Below we provide brief descriptions of how these tests work. 
+
+## Test descriptions
+
+The recommended set of tests that users run through the
+`oonideckgen` command include the following:
+
+**Web connectivity:** This test examines whether websites are reachable and if
+they are not, it attempts to determine whether access to them is blocked 
through
+DNS tampering, TCP connection RST/IP blocking or by having a transparent HTTP
+proxy. It does so by identifying the resolver of the user, performing a DNS
+lookup, attempting to establish a TCP session and by sending HTTP GET requests
+to the servers that are hosting tested websites.
+
+**HTTP invalid request line:** This test tries to detect the presence of 
network
+components (“middle box”) which could be responsible for censorship and/or
+traffic manipulation. Instead of sending a normal HTTP request, this test sends
+an invalid HTTP request line - containing an invalid HTTP version number, an
+invalid field count and a huge request method – to an echo service listening 
on
+the standard HTTP port. If a middle box is present in the tested network, the
+invalid HTTP request line will be intercepted by the middle box and this may
+trigger error messages which can help identify the proxy technologies.
+
+**HTTP header field manipulation:** This test tries to detect the presence of
+network components (“middle box”) which could be responsible for censorship
+and/or traffic manipulation. It does so by sending HTTP requests which include
+valid, but non-canonical HTTP headers to a backend control server which sends
+back any data it receives. If we receive the HTTP headers exactly as we sent
+them, then we assume that there is no “middle box” in the network. If,
+however, such software is present in the network that we are testing, it will
+likely normalize the invalid headers that we are sending or add extra headers.
+
+Another test which attempts to detect traffic manipulation includes **Multi-
+protocol traceroute**, which constructs packets in such a way that they perform
+a traceroute from multiple protocols and ports simultaneously. Other tests
+include **Tor bridge reachability**, **Psiphon**, **Lantern**, **OpenVPN** and
+**Meek fronted requests**, which examine whether these services work within a
+tested network by attempting to connect to them in an automated way.
+
+Further test descriptions can be found here.
+
+## Risks
+
+Many countries have a lengthy history of subjecting digital rights activists to
+various forms of abuse that could make it dangerous for individuals in these
+countries to run OONI. The use of OONI might therefore subject users to severe
+civil, criminal, or extra-judicial penalties, and such sanctions can 
potentially
+include:
+
+* Imprisonment
+
+* Physical assaults
+
+* Large fines
+
+* Receiving threats
+
+* Being placed on government watch lists
+
+* Targeted for surveillance
+
+While most countries don't have laws which specifically prohibit the use of
+network measurement software, it's important to note that the use of OONI can
+*still* 

[tor-commits] [ooni-probe/master] ignoreExistingDirectory is a recent twisted thing.

[art]

commit 21e6b839918e88f8260a58b23614194ec1230247
Author: Arturo Filastò 
Date:   Fri Aug 5 17:16:50 2016 +0200

ignoreExistingDirectory is a recent twisted thing.

Wrap calls to makedirs with proper exception handling.
---
 ooni/deck/store.py | 13 +++--
 ooni/resources.py  | 13 +++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 2d24f29..295f817 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -1,5 +1,6 @@
 import csv
 import json
+import errno
 from copy import deepcopy
 
 from twisted.internet import defer
@@ -76,8 +77,16 @@ class InputStore(object):
 self.path = FilePath(config.inputs_directory)
 self.resources = FilePath(config.resources_directory)
 
-self.path.child("descriptors").makedirs(ignoreExistingDirectory=True)
-self.path.child("data").makedirs(ignoreExistingDirectory=True)
+try:
+self.path.child("descriptors").makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
+try:
+self.path.child("data").makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 yield self.update_url_lists(country_code)
 
 @defer.inlineCallbacks
diff --git a/ooni/resources.py b/ooni/resources.py
index 9615c53..f97616c 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -1,4 +1,5 @@
 import json
+import errno
 
 from twisted.python.filepath import FilePath
 from twisted.internet import defer
@@ -107,7 +108,11 @@ def check_for_update(country_code=None):
 latest_version = yield get_latest_version()
 
 resources_dir = FilePath(config.resources_directory)
-resources_dir.makedirs(ignoreExistingDirectory=True)
+try:
+resources_dir.makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 current_manifest = resources_dir.child("manifest.json")
 
 if current_manifest.exists():
@@ -149,7 +154,11 @@ def check_for_update(country_code=None):
 filename = filename[:-3]
 gzipped = True
 dst_file = resources_dir.child(pre_path).child(filename)
-dst_file.parent().makedirs(ignoreExistingDirectory=True)
+try:
+dst_file.parent().makedirs()
+except OSError as e:
+if not e.errno == errno.EEXIST:
+raise
 src_file = dst_file.temporarySibling()
 src_file.alwaysCreate = 0
 



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix unittests for director, reporter and oonicli

[art]

commit 539c491e304a89ef2e57cb49aeeadbddab8d8baa
Author: Arturo Filastò 
Date:   Fri Aug 5 17:45:47 2016 +0200

Fix unittests for director, reporter and oonicli
---
 ooni/director.py|  1 -
 ooni/templates/httpt.py |  4 ++--
 ooni/tests/bases.py |  1 +
 ooni/tests/test_director.py |  5 +++--
 ooni/tests/test_oonicli.py  | 10 +++---
 ooni/tests/test_reporter.py |  4 +++-
 6 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 304a14a..ab31bf5 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -373,7 +373,6 @@ class Director(object):
 Launches a Tor with :param: socks_port :param: control_port
 :param: tor_binary set in ooniprobe.conf
 """
-from txtorcon import TorConfig
 if self._tor_state == 'running':
 log.debug("Tor is already running")
 defer.returnValue(self._tor_state)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index cba8702..d1aa1dc 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -110,8 +110,8 @@ class HTTPTest(NetTestCase):
 self.agent = FixedRedirectAgent(self.agent)
 self.report['agent'] = 'redirect'
 except:
-log.err("Warning! You are running an old version of twisted"\
-"(<= 10.1). I will not be able to follow redirects."\
+log.err("Warning! You are running an old version of twisted "
+"(<= 10.1). I will not be able to follow redirects."
 "This may make the testing less precise.")
 
 if len(self.contentDecoders) > 0:
diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py
index 40e3b5e..5a7da5f 100644
--- a/ooni/tests/bases.py
+++ b/ooni/tests/bases.py
@@ -11,6 +11,7 @@ class ConfigTestCase(unittest.TestCase):
 self.ooni_home_dir = os.path.abspath("ooni_home")
 self.config = config
 self.config.initialize_ooni_home(self.ooni_home_dir)
+config.is_initialized = lambda: True
 super(ConfigTestCase, self).setUp()
 
 def skipTest(self, reason):
diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py
index 6638adb..e5ca084 100644
--- a/ooni/tests/test_director.py
+++ b/ooni/tests/test_director.py
@@ -72,10 +72,11 @@ class TestDirector(ConfigTestCase):
 def test_start_tor(self):
 @defer.inlineCallbacks
 def director_start_tor():
+self.config.advanced.start_tor = True
 director = Director()
 yield director.start_tor()
-assert config.tor.socks_port == 4242
-assert config.tor.control_port == 4242
+self.assertEqual(config.tor.socks_port, 4242)
+self.assertEqual(config.tor.control_port, 4242)
 
 return director_start_tor()
 
diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index 06a21ff..8ad8a36 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -66,8 +66,6 @@ class TestRunDirector(ConfigTestCase):
 if not is_internet_connected():
 self.skipTest("You must be connected to the internet to run this 
test")
 
-config.tor.socks_port = 9050
-config.tor.control_port = None
 self.filenames = ['example-input.txt']
 with open('example-input.txt', 'w+') as f:
 f.write('http://torproject.org/\n')
@@ -83,7 +81,7 @@ class TestRunDirector(ConfigTestCase):
 
 @defer.inlineCallbacks
 def run_helper(self, test_name, nettest_args, verify_function, 
ooni_args=()):
-output_file = os.path.abspath('test_report.yamloo')
+output_file = os.path.abspath('test_report.njson')
 self.filenames.append(output_file)
 oldargv = sys.argv
 sys.argv = ['']
@@ -91,6 +89,12 @@ class TestRunDirector(ConfigTestCase):
 sys.argv.extend(['-n', '-o', output_file, test_name])
 sys.argv.extend(nettest_args)
 global_options = setupGlobalOptions(False, False, False)
+
+config.tor.socks_port = 9050
+config.advanced.start_tor = False
+config.tor.control_port = None
+config.advanced.debug = True
+
 yield runWithDirector(global_options,
   create_input_store=False)
 with open(output_file) as f:
diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py
index cbfdaeb..f7a917d 100644
--- a/ooni/tests/test_reporter.py
+++ b/ooni/tests/test_reporter.py
@@ -3,6 +3,7 @@ import yaml
 import json
 import time
 import shutil
+import tempfile
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -119,8 +120,8 @@ class TestOONIBReportLog(ConfigTestCase):
 
 def setUp(self):
 super(TestOONIBReportLog, self).setUp()
-self.report_log = OONIBReportLog()
 self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
+

[tor-commits] [ooni-probe/master] Use errno instead of integer to detect if file already exists

[art]

commit 1952ca0f1d55d8b78a1bed22c150bdfaf288b94e
Author: Arturo Filastò 
Date:   Thu Sep 1 17:30:45 2016 +0200

Use errno instead of integer to detect if file already exists
---
 ooni/settings.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 065252b..a0ef0a5 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import yaml
+import errno
 import getpass
 from ConfigParser import SafeConfigParser
 
@@ -332,7 +333,7 @@ class OConfig(object):
 try:
 os.makedirs(path)
 except OSError as exc:
-if exc.errno != 17:
+if exc.errno != errno.EEXIST:
 raise
 
 def create_config_file(self, include_ip=False, include_asn=True,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Improvements to start, stop, status workflow of ooniprobe-agent

[art]

commit e992f35ca6175ec53ec9966ca17966d86c129e8b
Author: Arturo Filastò 
Date:   Thu Sep 1 18:34:48 2016 +0200

Improvements to start, stop, status workflow of ooniprobe-agent

* Better support for running it across different users
---
 ooni/scripts/ooniprobe_agent.py | 64 +
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index ecfe7d8..f56f2df 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -2,6 +2,7 @@ from __future__ import print_function
 
 import os
 import time
+import errno
 import signal
 
 from twisted.scripts import twistd
@@ -55,7 +56,8 @@ def start_agent(options=None):
 
 # Since we are starting the logger below ourselves we make twistd log to
 #  a null log observer
-twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull']
+twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
+   '--umask', '022']
 twistd_config = OoniprobeTwistdConfig()
 if options is not None:
 twistd_args.extend(options.twistd_args)
@@ -68,9 +70,12 @@ def start_agent(options=None):
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
 
-if status_agent() == 0:
+try:
+get_running_pidfile()
 print("Stop ooniprobe-agent before attempting to start it")
 return 1
+except NotRunning:
+pass
 
 print("Starting ooniprobe agent.")
 WEB_UI_URL = "http://{0}:{1}".format(
@@ -80,8 +85,16 @@ def start_agent(options=None):
 twistd.runApp(twistd_config)
 return 0
 
-def status_agent():
-running = False
+
+class NotRunning(RuntimeError):
+pass
+
+def get_running_pidfile():
+"""
+:return: This pid of the running ooniprobe-agent instance.
+:raises: NotRunning if it's not running
+"""
+running_pidfile = None
 for pidfile in [config.system_pid_path, config.user_pid_path]:
 if not os.path.exists(pidfile):
 # Didn't find the pid_file
@@ -90,36 +103,51 @@ def status_agent():
 pid = int(pid)
 try:
 os.kill(pid, signal.SIG_DFL)
-running = True
-except OSError, oserr:
-if oserr.errno == 3:
+running_pidfile = pidfile
+break
+except OSError as ose:
+if ose.errno == errno.ESRCH:
 # Found pid, but isn't running
 continue
-if running is True:
+elif ose.errno == errno.EPERM:
+# The process is owned by root. We assume it's running
+running_pidfile = pidfile
+break
+if running_pidfile is None:
+raise NotRunning
+return running_pidfile
+
+def status_agent():
+try:
+get_running_pidfile()
 print("ooniprobe-agent is running")
 return 0
-print("ooniprobe-agent is NOT running")
-return 1
+except NotRunning:
+print("ooniprobe-agent is NOT running")
+return 1
 
 def stop_agent():
 # This function is borrowed from tahoe
-pidfile = os.path.join(
-config.running_path,
-'twistd.pid'
-)
-if not os.path.exists(pidfile):
-print("It seems like ooniprobe-agent is not running")
+try:
+pidfile = get_running_pidfile()
+except NotRunning:
+print("ooniprobe-agent is NOT running. Nothing to do.")
 return 2
+
 pid = open(pidfile, "r").read()
 pid = int(pid)
 try:
 os.kill(pid, signal.SIGKILL)
-except OSError, oserr:
-if oserr.errno == 3:
+except OSError as ose:
+if ose.errno == errno.ESRCH:
 print("No process was running. Cleaning up.")
 # the process didn't exist, so wipe the pid file
 os.remove(pidfile)
 return 2
+elif ose.errno == errno.EPERM:
+# The process is owned by root. We assume it's running
+print("ooniprobe-agent is owned by root. We cannot stop it.")
+return 3
 else:
 raise
 try:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge branch 'master' into v2.0.0-alpha

[art]

commit 847a951b40d1355c7f18d8e91001ad9a1e5fc984
Merge: 8048dbd da00195
Author: Arturo Filastò 
Date:   Fri Aug 26 18:33:38 2016 +0200

Merge branch 'master' into v2.0.0-alpha

* master:
  Don't look for the namebench csv file since we don't ship it




___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add support for initialization of ooniprobe

[art]

commit 27a299c5c7d195860e63aa0b1d316f3255e4857d
Author: Arturo Filastò 
Date:   Wed Aug 3 13:05:31 2016 +0200

Add support for initialization of ooniprobe
---
 data/decks/web.yaml   |   1 +
 ooni/agent/scheduler.py   |  25 +++---
 ooni/scripts/ooniprobe.py |   4 +-
 ooni/settings.py  | 191 ++
 ooni/tests/__init__.py|   7 --
 ooni/ui/cli.py|  52 +++-
 ooni/ui/web/client/index.html |   2 +-
 ooni/ui/web/server.py |  82 +-
 ooni/utils/__init__.py|   1 -
 9 files changed, 329 insertions(+), 36 deletions(-)

diff --git a/data/decks/web.yaml b/data/decks/web.yaml
index a81b8f8..c7b9bdc 100644
--- a/data/decks/web.yaml
+++ b/data/decks/web.yaml
@@ -2,6 +2,7 @@
 name: Web related ooniprobe tests
 description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
 Request and the Web Connectivity test
+schedule: "@daily"
 tasks:
 - name: Runs the HTTP Header Field Manipulation test
   ooni:
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index a6d689f..1f51bd4 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -22,9 +22,11 @@ class ScheduledTask(object):
 schedule = None
 identifier = None
 
-def __init__(self, schedule=None):
+def __init__(self, schedule=None, identifier=None):
 if schedule is not None:
 self.schedule = schedule
+if identifier is not None:
+self.identifier = identifier
 
 assert self.identifier is not None, "self.identifier must be set"
 assert self.schedule is not None, "self.schedule must be set"
@@ -120,23 +122,23 @@ class DeleteOldReports(ScheduledTask):
 measurement_path.child(measurement['id']).remove()
 
 
-class RunDecks(ScheduledTask):
+class RunDeck(ScheduledTask):
 """
 This will run the decks that have been configured on the system as the
 decks to run by default.
 """
-schedule = '@daily'
-identifier = 'run-decks'
 
-def __init__(self, director, schedule=None):
-super(RunDecks, self).__init__(schedule)
+def __init__(self, director, deck_id, schedule):
+self.deck_id = deck_id
 self.director = director
+identifier = 'run-deck-' + deck_id
+super(RunDeck, self).__init__(schedule, identifier)
 
 @defer.inlineCallbacks
 def task(self):
-for deck_id, deck in deck_store.list_enabled():
-yield deck.setup()
-yield deck.run(self.director)
+deck = deck_store.get(self.deck_id)
+yield deck.setup()
+yield deck.run(self.director)
 
 class SendHeartBeat(ScheduledTask):
 """
@@ -215,7 +217,10 @@ class SchedulerService(service.MultiService):
 self.schedule(UpdateInputsAndResources())
 self.schedule(UploadReports())
 self.schedule(DeleteOldReports())
-self.schedule(RunDecks(self.director))
+for deck_id, deck in deck_store.list_enabled():
+if deck.schedule is None:
+continue
+self.schedule(RunDeck(self.director, deck_id, deck.schedule))
 
 self._looping_call.start(self.interval)
 
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index f5d5b59..430252a 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -6,12 +6,14 @@ from twisted.internet import task, defer
 
 def ooniprobe(reactor):
 from ooni.ui.cli import runWithDaemonDirector, runWithDirector
-from ooni.ui.cli import setupGlobalOptions
+from ooni.ui.cli import setupGlobalOptions, initializeOoniprobe
 
 global_options = setupGlobalOptions(logging=True, start_tor=True,
 check_incoherences=True)
 if global_options['queue']:
 return runWithDaemonDirector(global_options)
+elif global_options['initialize']:
+return initializeOoniprobe(global_options)
 elif global_options['web-ui']:
 from ooni.scripts.ooniprobe_agent import WEB_UI_URL
 from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/settings.py b/ooni/settings.py
index 2161560..8bb3340 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,6 +13,125 @@ from ooni.utils.net import ConnectAndCloseProtocol, 
connectProtocol
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
+
+CONFIG_FILE_TEMPLATE = """\
+# This is the configuration file for OONIProbe
+# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
+# Keep in mind that indentation matters.
+
+basic:
+# Where OONIProbe should be writing it's log file
+logfile: {logfile}
+loglevel: WARNING
+privacy:
+# Should we include the IP address of the probe in the report?
+includeip: {include_ip}
+# Should we include the ASN of the probe in the report?
+includeasn: {include_asn}
+# Should we 

[tor-commits] [ooni-probe/master] Implement disk quota management

[art]

commit def4c929d162852ace8a016bb3352677eec5bcde
Author: Arturo Filastò 
Date:   Tue Aug 30 02:09:55 2016 +0200

Implement disk quota management

This is related to the feature described in here: 
https://github.com/TheTorProject/lepidopter/issues/53
---
 ooni/agent/scheduler.py  | 67 
 ooni/measurements.py | 20 +++
 ooni/settings.py |  6 -
 ooni/tests/test_utils.py | 34 
 ooni/ui/web/server.py| 12 -
 ooni/utils/files.py  | 34 
 6 files changed, 166 insertions(+), 7 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 3389db1..54551f6 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,3 +1,6 @@
+import os
+import errno
+
 from datetime import datetime
 
 from twisted.application import service
@@ -8,6 +11,7 @@ from twisted.python.filepath import FilePath
 from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
+from ooni.utils.files import human_size_to_bytes, directory_usage
 from ooni.deck.store import input_store, deck_store
 from ooni.settings import config
 from ooni.contrib import croniter
@@ -146,6 +150,66 @@ class DeleteOldReports(ScheduledTask):
 measurement_path.child(measurement['id']).remove()
 
 
+class CheckMeasurementQuota(ScheduledTask):
+"""
+This task is run to ensure we don't run out of disk space and deletes
+older reports to avoid filling the quota.
+"""
+identifier = 'check-measurement-quota'
+schedule = '@hourly'
+_warn_when = 0.8
+
+def task(self):
+if config.basic.measurement_quota is None:
+return
+maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
+available_bytes = directory_usage(config.measurements_directory)
+warning_path = os.path.join(config.running_path, 'quota_warning')
+
+if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when:
+log.warn("You are about to reach the maximum allowed quota. Be 
careful")
+with open(warning_path, "w") as out_file:
+out_file.write("{0} {1}".split(available_bytes, maximum_bytes))
+else:
+try:
+os.remove(warning_path)
+except OSError as ose:
+if ose.errno != errno.ENOENT:
+raise
+
+if float(available_bytes) < float(maximum_bytes):
+# We are within the allow quota exit.
+return
+
+# We should begin to delete old reports
+amount_to_delete = float(maximum_bytes) - float(available_bytes)
+amount_deleted = 0
+measurement_path = FilePath(config.measurements_directory)
+
+kept_measurements = []
+stale_measurements = []
+remaining_measurements = []
+measurements_by_date = sorted(list_measurements(compute_size=True),
+  key=lambda k: k['test_start_time'])
+for measurement in measurements_by_date:
+if measurement['keep'] is True:
+kept_measurements.append(measurement)
+elif measurement['stale'] is True:
+stale_measurements.append(measurement)
+else:
+remaining_measurements.append(measurement)
+
+# This is the order in which we should begin deleting measurements.
+ordered_measurements = (stale_measurements +
+remaining_measurements +
+kept_measurements)
+while amount_deleted < amount_to_delete:
+measurement = ordered_measurements.pop(0)
+log.warn("Deleting report {0}".format(measurement["id"]))
+measurement_path.child(measurement['id']).remove()
+amount_deleted += measurement['size']
+
+
 class RunDeck(ScheduledTask):
 """
 This will run the decks that have been configured on the system as the
@@ -196,6 +260,7 @@ SYSTEM_TASKS = [
 UpdateInputsAndResources
 ]
 
+
 @defer.inlineCallbacks
 def run_system_tasks(no_input_store=False):
 task_classes = SYSTEM_TASKS[:]
@@ -215,6 +280,7 @@ def run_system_tasks(no_input_store=False):
 log.err("Failed to run task {0}".format(task.identifier))
 log.exception(exc)
 
+
 class SchedulerService(service.MultiService):
 """
 This service is responsible for running the periodic tasks.
@@ -271,6 +337,7 @@ class SchedulerService(service.MultiService):
 self.schedule(UpdateInputsAndResources())
 self.schedule(UploadReports())
 self.schedule(DeleteOldReports())
+self.schedule(CheckMeasurementQuota())
 self.schedule(RefreshDeckList(self))
 
 self._looping_call.start(self.interval)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index 6d90c1b..fd722ee 

[tor-commits] [ooni-probe/master] Bump to alpha.2

[art]

commit f3d3bb10d52e472c880eaa0ca3f69cc3fe14dc6d
Author: Arturo Filastò 
Date:   Mon Aug 8 16:42:16 2016 +0200

Bump to alpha.2

* Use semantic versioning
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index 1a31608..9eecb67 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a1"
+__version__ = "2.0.0-alpha.2"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] settings.py: make sure default paths are okay

[art]

commit 8048dbd1d1775dc658537b138e102a9e8319f147
Author: Simone Basso 
Date:   Mon Aug 8 19:24:29 2016 +0200

settings.py: make sure default paths are okay
---
 ooni/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 16de94d..3da1da7 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -143,8 +143,8 @@ if hasattr(sys, 'real_prefix'):
 # These are the the embedded settings
 _SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini')
 
-USR_SHARE_PATH = '/var/lib/ooni'
-VAR_LIB_PATH = '/usr/share/ooni'
+USR_SHARE_PATH = '/usr/share/ooni'
+VAR_LIB_PATH = '/var/lib/ooni'
 ETC_PATH = '/etc'
 
 if IS_VIRTUALENV:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Ignore global options that are None or set to zero.

[art]

commit 196fc17c2d62a8eb37e95e55f0dedaa2c53284f5
Author: Arturo Filastò 
Date:   Mon Aug 29 09:39:14 2016 +0200

Ignore global options that are None or set to zero.

* Always pop values from task_data to ensure the options are always clean.

This closes #592
---
 ooni/deck/deck.py   | 16 ++--
 ooni/tests/test_deck.py | 10 +-
 ooni/ui/cli.py  |  4 ++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 037c5d6..e4c35c4 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -317,12 +317,16 @@ class DeckTask(object):
 
 self._load(data)
 
-def _get_option(self, name, task_data, default=None):
+def _pop_option(self, name, task_data, default=None):
 try:
-return self.global_options[name]
+value = self.global_options[name]
+if value in [None, 0]:
+raise KeyError
 except KeyError:
-return task_data.pop(name,
- self.parent_metadata.get(name, default))
+value = task_data.pop(name,
+  self.parent_metadata.get(name, default))
+task_data.pop(name, None)
+return value
 
 def _load_ooni(self, task_data):
 required_keys = ["test_name"]
@@ -334,8 +338,8 @@ class DeckTask(object):
 nettest_path = nettest_to_path(task_data.pop("test_name"),
self._arbitrary_paths)
 
-annotations = self._get_option('annotations', task_data, {})
-collector_address = self._get_option('collector', task_data, None)
+annotations = self._pop_option('annotations', task_data, {})
+collector_address = self._pop_option('collector', task_data, None)
 
 try:
 self.output_path = self.global_options['reportfile']
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 1bc6097..b2d05a6 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -14,7 +14,7 @@ from hashlib import sha256
 from ooni import errors
 from ooni.deck.store import input_store
 from ooni.deck.backend import lookup_collector_and_test_helpers
-from ooni.deck.deck import nettest_to_path, NGDeck
+from ooni.deck.deck import nettest_to_path, NGDeck, options_to_args
 from ooni.deck.legacy import convert_legacy_deck
 from ooni.tests.bases import ConfigTestCase
 from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
@@ -330,3 +330,11 @@ class TestNGDeck(ConfigTestCase):
 "manipulation/http_header_field_manipulation",
 "blocking/web_connectivity"
 ])
+tasks = map(lambda task: task['ooni'], ng_deck['tasks'])
+self.assertEqual(
+tasks[2]['f'],
+'/path/to/citizenlab-urls-global.txt')
+
+def test_options_to_args(self):
+args = options_to_args({"f": "foobar.txt", "bar": None, "help": 0})
+print(args)
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 61f254b..3ad9605 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -338,8 +338,8 @@ def createDeck(global_options, url=None):
 raise SystemExit(3)
 
 except errors.OONIUsageError as e:
-log.err(e)
-print e.net_test_loader.usageOptions().getUsage()
+log.exception(e)
+map(log.msg, e.net_test_loader.usageOptions().getUsage().split("\n"))
 raise SystemExit(4)
 
 except errors.HTTPSCollectorUnsupported:



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Merge pull request #581 from TheTorProject/feature/webui2-rb

[art]

commit 0b1ffa329ee7db8e11131f59f2113c9e55de3ee4
Merge: eaf15e6 f3d3bb1
Author: Arturo Filastò 
Date:   Mon Aug 8 16:43:59 2016 +0200

Merge pull request #581 from TheTorProject/feature/webui2-rb

Feature/webui2 rb

 .gitignore |1 +
 MANIFEST.in|3 +
 Vagrantfile|   33 +-
 bin/Makefile   |   81 --
 bin/oonid  |7 -
 bin/oonideckgen|   37 -
 bin/ooniprobe  |   24 -
 bin/ooniprobe-dev  |   10 -
 bin/oonireport |   38 -
 bin/ooniresources  |   35 -
 bin/test/ooni/__init__.py  |1 -
 data/component-template/templ.component.js |   10 -
 data/component-template/templ.controller.js|5 -
 data/component-template/templ.css  |0
 data/component-template/templ.html |8 -
 data/component-template/templ.js   |   19 -
 data/decks/web-full.yaml   |   23 +
 data/decks/web-no-invalid.yaml |   18 +
 data/ooniprobe.conf.sample |5 +-
 ooni/__init__.py   |   13 +-
 ooni/agent/__init__.py |0
 ooni/agent/agent.py|   27 +
 ooni/agent/scheduler.py|  280 
 ooni/backend_client.py |  117 +-
 ooni/constants.py  |4 +-
 ooni/contrib/__init__.py   |1 +
 ooni/contrib/croniter.py   |  430 +++
 ooni/contrib/dateutil/__init__.py  |0
 ooni/contrib/dateutil/relativedelta.py |  539 
 ooni/contrib/dateutil/tz/__init__.py   |4 +
 ooni/contrib/dateutil/tz/_common.py|  100 ++
 ooni/contrib/dateutil/tz/tz.py | 1339 
 ooni/contrib/dateutil/tz/win.py|  354 ++
 ooni/deck.py   |  413 --
 ooni/deck/__init__.py  |1 +
 ooni/deck/backend.py   |  191 +++
 ooni/deck/deck.py  |  405 ++
 ooni/deck/legacy.py|   65 +
 ooni/deck/store.py |  192 +++
 ooni/deckgen/__init__.py   |1 -
 ooni/deckgen/cli.py|  183 ---
 ooni/deckgen/processors/__init__.py|0
 ooni/deckgen/processors/citizenlab_test_lists.py   |   55 -
 ooni/deckgen/processors/namebench_dns_servers.py   |   51 -
 ooni/director.py   |  227 ++--
 ooni/geoip.py  |   57 +-
 ooni/measurements.py   |  151 ++-
 ooni/nettest.py|  118 +-
 ooni/nettests/blocking/web_connectivity.py |   14 +-
 .../manipulation/http_invalid_request_line.py  |   14 +-
 ooni/oonicli.py|  520 
 ooni/otime.py  |7 +
 ooni/report/__init__.py|1 -
 ooni/report/cli.py |   90 --
 ooni/report/parser.py  |   35 -
 ooni/report/tool.py|  117 --
 ooni/reporter.py   |  428 ---
 ooni/resources.py  |  184 +++
 ooni/resources/__init__.py |7 -
 ooni/resources/cli.py  |   44 -
 ooni/resources/update.py   |   57 -
 ooni/scripts/__init__.py   |0
 ooni/scripts/oonideckgen.py|  138 ++
 ooni/scripts/ooniprobe.py  |   35 +
 ooni/scripts/ooniprobe_agent.py|  165 +++
 ooni/scripts/oonireport.py |  309 +
 ooni/scripts/ooniresources.py  |   34 +
 ooni/settings.py   |  364 --
 ooni/templates/httpt.py|9 +-
 ooni/templates/process.py  |7 +-
 ooni/templates/tcpt.py |1 -
 ooni/tests/__init__.py |7 -
 ooni/tests/bases.py|4 +-
 ooni/tests/mocks.py|2 +-
 ooni/tests/test_backend_client.py  |   16 -
 ooni/tests/test_deck.py|  271 ++--
 

[tor-commits] [ooni-probe/master] Add support for deck lifecycle in the web UI

[art]

commit 7663106d25d272ffc7ff2208d95cddef3b685f62
Author: Arturo Filastò 
Date:   Thu Aug 4 19:46:26 2016 +0200

Add support for deck lifecycle in the web UI

* Fix a series of bugs found while testing
* Bump the version number up one
---
 MANIFEST.in|   1 +
 data/decks/web-full.yaml   |  23 ++
 data/decks/web-no-invalid.yaml |  18 +
 data/decks/web.yaml|  23 --
 ooni/__init__.py   |   2 +-
 ooni/agent/agent.py|   9 ++-
 ooni/agent/scheduler.py|  64 ++---
 ooni/deck/deck.py  |   9 ++-
 ooni/deck/store.py |   4 +-
 ooni/resources.py  |   5 ++
 ooni/scripts/oonideckgen.py|  20 ++
 ooni/scripts/ooniprobe.py  |   3 +-
 ooni/scripts/oonireport.py |   7 +-
 ooni/settings.py   | 154 +++--
 ooni/ui/cli.py |  37 --
 ooni/ui/web/client/index.html  |   2 +-
 ooni/ui/web/server.py  |  28 +++-
 ooni/ui/web/web.py |   5 +-
 ooni/utils/__init__.py |   4 +-
 setup.py   |  29 +++-
 20 files changed, 271 insertions(+), 176 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 485e834..60d2ef9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,3 +9,4 @@ include data/ooniprobe.conf.sample
 include data/configs/lepidopter-ooniprobe.conf
 include data/configs/lepidopter-oonireport.conf
 include ooni/settings.ini
+include ooni/ui/consent-form.md
diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
new file mode 100644
index 000..7812505
--- /dev/null
+++ b/data/decks/web-full.yaml
@@ -0,0 +1,23 @@
+---
+name: Full Web test deck
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+Request and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+  ooni:
+test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web-no-invalid.yaml b/data/decks/web-no-invalid.yaml
new file mode 100644
index 000..ea93488
--- /dev/null
+++ b/data/decks/web-no-invalid.yaml
@@ -0,0 +1,18 @@
+---
+name: Web test deck without HTTP Invalid Request Line
+description: This deck runs HTTP Header Field Manipulation, and the Web 
Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+test_name: http_header_field_manipulation
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
deleted file mode 100644
index c7b9bdc..000
--- a/data/decks/web.yaml
+++ /dev/null
@@ -1,23 +0,0 @@

-name: Web related ooniprobe tests
-description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
-Request and the Web Connectivity test
-schedule: "@daily"
-tasks:
-- name: Runs the HTTP Header Field Manipulation test
-  ooni:
-test_name: http_header_field_manipulation
-
-- name: Runs the HTTP Invalid Request Line test
-  ooni:
-test_name: http_invalid_request_line
-
-- name: Runs the Web Connectivity Test
-  ooni:
-test_name: web_connectivity
-file: $citizenlab_global_urls
-
-- name: Runs the Web Connectivity Test
-  ooni:
-test_name: web_connectivity
-file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 653a636..1a31608 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a0"
+__version__ = "2.0.0a1"
 
 __all__ = [
 'agent',
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index c2e7e26..0311cef 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -11,12 +11,15 @@ class AgentService(service.MultiService):
 
 director = Director()
 
-self.web_ui_service = WebUIService(director, web_ui_port)
-self.web_ui_service.setServiceParent(self)
-
 self.scheduler_service = SchedulerService(director)
 self.scheduler_service.setServiceParent(self)
 
+self.web_ui_service = WebUIService(director,
+   self.scheduler_service,
+   web_ui_port)
+self.web_ui_service.setServiceParent(self)
+
+
 def startService(self):
 service.MultiService.startService(self)
 
diff --git 

[tor-commits] [ooni-probe/master] Don't look for the namebench csv file since we don't ship it

[art]

commit da00195094cdf22678d42608cdbc3bb22b9d757e
Author: Arturo Filastò 
Date:   Wed Aug 10 18:11:06 2016 +0200

Don't look for the namebench csv file since we don't ship it

* Include an empty file so that older ooniprobes don't end up always 
updating
  the resources on every cycle.
---
 ooni/deckgen/cli.py | 4 
 setup.py| 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index a6046f3..27e465a 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -122,10 +122,6 @@ def resources_up_to_date():
 return False
 
 if config.get_data_file_path("resources/"
- "namebench-dns-servers.csv") is None:
-return False
-
-if config.get_data_file_path("resources/"
  "citizenlab-test-lists/"
  "global.csv") is None:
 return False
diff --git a/setup.py b/setup.py
index ff1b6c2..b175662 100644
--- a/setup.py
+++ b/setup.py
@@ -279,6 +279,9 @@ class CreateOoniResources(Command):
 pj(pkg_dir, "GeoIP", "GeoIPASNum.dat"))
 shutil.move(pj(tmp_dir, "test-lists-master", "lists"),
 pj(pkg_dir, "resources", "citizenlab-test-lists"))
+# Touch the namebench dns servers file
+with open(pj(pkg_dir, "resources", "namebench-dns-servers.csv"), "w"):
+pass
 # Don't include services and official lists
 shutil.rmtree(
 pj(pkg_dir,



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Run the check_for_update function when installation is complete

[art]

commit f7fec4b6d0aca457cfafdb0617218f2b4c0f0345
Author: Arturo Filastò 
Date:   Sat Jul 30 17:53:32 2016 +0200

Run the check_for_update function when installation is complete
---
 Vagrantfile |   2 +-
 ooni/agent/scheduler.py |   6 ++-
 setup.py| 114 +++-
 3 files changed, 10 insertions(+), 112 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index cf73244..0e75d55 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -13,7 +13,7 @@ apt-get update
 apt-get install -y tor deb.torproject.org-keyring
 
 # Setup for sniffer subsystem
-apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev 
libffi-dev python-dev python-pip
+apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev 
libffi-dev python-dev python-pip libssl-dev
 cd /data/ooni-probe
 python setup.py install
 
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 7a77afb..0d8689c 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -1,7 +1,8 @@
 from datetime import datetime
 
 from twisted.application import service
-from twisted.internet import task, defer
+from twisted.internet import defer
+from twisted.internet.task import LoopingCall
 from twisted.python.filepath import FilePath
 
 from ooni.scripts import oonireport
@@ -72,6 +73,7 @@ class ScheduledTask(object):
 finally:
 self._last_run_lock.unlock()
 
+
 class UpdateInputsAndResources(ScheduledTask):
 identifier = "update-inputs"
 schedule = "@daily"
@@ -177,7 +179,7 @@ class SchedulerService(service.MultiService):
 service.MultiService.__init__(self)
 self.director = director
 self.interval = interval
-self._looping_call = task.LoopingCall(self._should_run)
+self._looping_call = LoopingCall(self._should_run)
 self._scheduled_tasks = []
 
 def schedule(self, task):
diff --git a/setup.py b/setup.py
index 6940f66..da48323 100644
--- a/setup.py
+++ b/setup.py
@@ -89,16 +89,14 @@ Have fun!
 from __future__ import print_function
 
 import os
-import sys
 import shutil
 import tempfile
 import subprocess
 from ConfigParser import SafeConfigParser
 
 from os.path import join as pj
-from setuptools import setup, Command
+from setuptools import setup
 from setuptools.command.install import install
-from distutils.spawn import find_executable
 
 from ooni import __version__, __author__
 
@@ -177,11 +175,9 @@ class OoniInstall(install):
 pass
 
 def ooniresources(self):
-ooniresources = find_executable("ooniresources")
-process = subprocess.Popen([ooniresources],
-   stdout=sys.stdout.fileno(),
-   stderr=sys.stderr.fileno())
-process.wait()
+from ooni.resources import check_for_update
+from twisted.internet import task
+task.react(lambda _: check_for_update())
 
 def update_lepidopter_config(self):
 try:
@@ -200,105 +196,6 @@ class OoniInstall(install):
 if is_lepidopter():
 self.update_lepidopter_config()
 
-class ExecutableNotFound(Exception):
-pass
-
-class CreateOoniResources(Command):
-description = ("Create ooni-resources.tar.gz containing test-lists and "
-   "GeoIP data files")
-user_options = []
-
-def initialize_options(self):
-pass
-def finalize_options(self):
-pass
-def download(self, url, directory, filename):
-dst_path = pj(directory, filename)
-args = [
-self.wget,
-"-O",
-dst_path,
-url
-]
-out = run_command(args)
-if out is None:
-raise Exception("Failed to download {0}".format(url))
-return dst_path
-
-def find_executables(self):
-self.wget = find_executable("wget")
-if not self.wget:
-raise ExecutableNotFound("wget")
-self.tar = find_executable("tar")
-if not self.tar:
-raise ExecutableNotFound("tar")
-self.unzip = find_executable("unzip")
-if not self.unzip:
-raise ExecutableNotFound("unzip")
-self.gunzip = find_executable("gunzip")
-if not self.gunzip:
-raise ExecutableNotFound("gunzip")
-
-def run(self):
-dst_path = "dist/ooni-resources.tar.gz"
-
-try:
-self.find_executables()
-except ExecutableNotFound as enf:
-print("ERR: Could not find '{0}'".format(enf.message))
-return
-
-tmp_dir = tempfile.mkdtemp()
-pkg_dir = tempfile.mkdtemp()
-
-os.mkdir(pj(pkg_dir, "resources"))
-os.mkdir(pj(pkg_dir, "GeoIP"))
-
-try:
-geoip_asn_path = self.download(GEOIP_ASN_URL, tmp_dir, 
"GeoIPASNum.dat.gz")
-except Exception as exc:
-print(exc.message)
-return
-try:
-

[tor-commits] [ooni-probe/master] Bump to version a0

[art]

commit d1765a80349cc5eba46b342056cc08cc29ae6ae0
Author: Arturo Filastò 
Date:   Sat Jul 30 17:23:46 2016 +0200

Bump to version a0
---
 ooni/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index c5c3c0a..653a636 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0.dev1"
+__version__ = "2.0.0a0"
 
 __all__ = [
 'agent',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Bind by default to localhost. Add support for disabling the web UI.

[art]

commit e020064d31ee93e85fda73dc7a89ac55b97f4aa4
Author: Arturo Filastò 
Date:   Fri Aug 26 18:44:53 2016 +0200

Bind by default to localhost. Add support for disabling the web UI.

* Expose these options via configuration options
This fixes #589
---
 ooni/agent/agent.py |  9 +
 ooni/scripts/ooniprobe_agent.py |  6 +++---
 ooni/settings.py| 10 +-
 ooni/ui/web/web.py  |  3 ++-
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index 0311cef..01e2162 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -14,10 +14,11 @@ class AgentService(service.MultiService):
 self.scheduler_service = SchedulerService(director)
 self.scheduler_service.setServiceParent(self)
 
-self.web_ui_service = WebUIService(director,
-   self.scheduler_service,
-   web_ui_port)
-self.web_ui_service.setServiceParent(self)
+if not config.advanced.disabled_webui:
+self.web_ui_service = WebUIService(director,
+   self.scheduler_service,
+   web_ui_port)
+self.web_ui_service.setServiceParent(self)
 
 
 def startService(self):
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 30df9b9..08fba89 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -11,14 +11,12 @@ from ooni.utils import log
 from ooni.settings import config
 from ooni.agent.agent import AgentService
 
-WEB_UI_PORT = 8842
-WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT)
 
 class StartOoniprobeAgentPlugin:
 tapname = "ooniprobe"
 
 def makeService(self, so):
-return AgentService(WEB_UI_PORT)
+return AgentService(config.advanced.webui_port)
 
 class OoniprobeTwistdConfig(twistd.ServerOptions):
 subCommands = [
@@ -70,6 +68,8 @@ def start_agent(options=None):
 "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
 }
 print("Starting ooniprobe agent.")
+WEB_UI_URL = "http://{0}:{1}".format(
+config.advanced.webui_address, config.advanced.webui_port)
 print("To view the GUI go to %s" % WEB_UI_URL)
 log.start()
 twistd.runApp(twistd_config)
diff --git a/ooni/settings.py b/ooni/settings.py
index 3da1da7..2bacd57 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -65,6 +65,11 @@ advanced:
 # insecure_backend: false
 # The preferred backend type, can be one of onion, https or cloudfront
 preferred_backend: {preferred_backend}
+# The port and address for the Web UI
+#webui_port: 8842
+#webui_address: "127.0.0.1"
+# Should the Web UI be disabled
+#disable_webui: false
 tor:
 #socks_port: 8801
 #control_port: 8802
@@ -121,7 +126,10 @@ defaults = {
 "reporting_retries": 5,
 "reporting_concurrency": 7,
 "insecure_backend": False,
-"preferred_backend": "onion"
+"preferred_backend": "onion",
+"webui_port": 8842,
+"webui_address": "127.0.0.1",
+"webui_disabled": False
 },
 "tor": {
 "socks_port": None,
diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py
index 10bbef1..f36e0a2 100644
--- a/ooni/ui/web/web.py
+++ b/ooni/ui/web/web.py
@@ -19,7 +19,8 @@ class WebUIService(service.MultiService):
 web_ui_api = WebUIAPI(config, self.director, self.scheduler)
 self._port = reactor.listenTCP(
 self.port_number,
-server.Site(web_ui_api.app.resource())
+server.Site(web_ui_api.app.resource()),
+interface=config.advanced.webui_address
 )
 
 def stopService(self):



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Convert the director status into a property so the IP and ASN get's updated

[art]

commit c7970d24f0285e08a57a3b362889f308dbcdbf3c
Author: Arturo Filastò 
Date:   Mon Aug 1 19:02:23 2016 +0200

Convert the director status into a property so the IP and ASN get's updated

* Make another call to resolveGeodata after we fetch the inputs

(this should fix the issue @agrabeli reported in erroneous IP lookups)
---
 ooni/agent/scheduler.py |  1 +
 ooni/ui/web/server.py   | 22 --
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 0d8689c..a6d689f 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -84,6 +84,7 @@ class UpdateInputsAndResources(ScheduledTask):
 yield probe_ip.lookup()
 yield resources.check_for_update(probe_ip.geodata['countrycode'])
 yield input_store.update(probe_ip.geodata['countrycode'])
+yield probe_ip.resolveGeodata()
 
 
 class UploadReports(ScheduledTask):
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 50db3ad..a03daa4 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -127,13 +127,7 @@ class WebUIAPI(object):
 self._xsrf_token = ''.join([rng.choice(token_space)
 for _ in range(30)])
 
-self.status = {
-"software_version": ooniprobe_version,
-"software_name": "ooniprobe",
-"asn": probe_ip.geodata['asn'],
-"country_code": probe_ip.geodata['countrycode'],
-"director_started": False
-}
+self._director_started = False
 
 self.status_poller = LongPoller(
 self._long_polling_timeout, _reactor)
@@ -150,14 +144,22 @@ class WebUIAPI(object):
 d.addCallback(self.director_started)
 d.addBoth(lambda _: self.status_poller.notify())
 
+@property
+def status(self):
+return {
+"software_version": ooniprobe_version,
+"software_name": "ooniprobe",
+"asn": probe_ip.geodata['asn'],
+"country_code": probe_ip.geodata['countrycode'],
+"director_started": self._director_started
+}
+
 def handle_director_event(self, event):
 log.msg("Handling event {0}".format(event.type))
 self.director_event_poller.notify(event)
 
 def director_started(self, _):
-self.status['director_started'] = True
-self.status["asn"] = probe_ip.geodata['asn']
-self.status["country_code"] = probe_ip.geodata['countrycode']
+self._director_started = True
 
 @app.handle_errors(NotFound)
 @xsrf_protect(check=False)



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Don't raise an exception if a deck is already enabled

[art]

commit 886e19ec52e96a52c4a1d34d8126337eea18d66c
Author: Arturo Filastò 
Date:   Wed Aug 31 19:42:58 2016 +0200

Don't raise an exception if a deck is already enabled

* Disable debug level logging in web server
---
 ooni/deck/store.py| 6 +-
 ooni/ui/web/server.py | 2 --
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 295f817..c938d56 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -162,7 +162,11 @@ class DeckStore(object):
 if not deck_path.exists():
 raise DeckNotFound(deck_id)
 deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
-deck_path.linkTo(deck_enabled_path)
+try:
+deck_path.linkTo(deck_enabled_path)
+except OSError as ose:
+if ose.errno != errno.EEXIST:
+raise
 
 def disable(self, deck_id):
 deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 2e89370..26bfd47 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -26,8 +26,6 @@ from ooni.measurements import get_summary, get_measurement, 
list_measurements
 from ooni.measurements import MeasurementNotFound, MeasurementInProgress
 from ooni.geoip import probe_ip
 
-config.advanced.debug = True
-
 class WebUIError(Exception):
 def __init__(self, code, message):
 self.code = code



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Add missing modules to setup.py

[art]

commit f5c02d1593b37d5fd54ab7671b0914b9ad3a78ed
Author: Arturo Filastò 
Date:   Sat Jul 30 00:26:29 2016 +0200

Add missing modules to setup.py
---
 setup.py | 4 
 1 file changed, 4 insertions(+)

diff --git a/setup.py b/setup.py
index c1ae189..6940f66 100644
--- a/setup.py
+++ b/setup.py
@@ -307,6 +307,10 @@ packages = [
 'ooni',
 'ooni.agent',
 'ooni.common',
+'ooni.contrib',
+'ooni.contrib.dateutil',
+'ooni.contrib.dateutil.tz',
+'ooni.deck',
 'ooni.kit',
 'ooni.nettests',
 'ooni.nettests.manipulation',



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Fix unittests of oonireport

[art]

commit e490533b48c7c62da5e6d28832434dbc51fb85f0
Author: Arturo Filastò 
Date:   Fri Jul 29 23:49:32 2016 +0200

Fix unittests of oonireport
---
 ooni/tests/test_oonireport.py | 19 +++
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/ooni/tests/test_oonireport.py b/ooni/tests/test_oonireport.py
index 4769682..fe5e420 100644
--- a/ooni/tests/test_oonireport.py
+++ b/ooni/tests/test_oonireport.py
@@ -1,5 +1,3 @@
-import yaml
-
 from mock import patch, MagicMock
 
 from twisted.internet import defer
@@ -9,15 +7,6 @@ mock_tor_check = MagicMock(return_value=True)
 
 class TestOONIReport(ConfigTestCase):
 
-def _create_reporting_yaml(self, filename):
-from ooni.settings import config
-with open(config.report_log_file, 'w+') as f:
-yaml.dump({
-filename: {
-"collector": "httpo://thirteenchars123.onion"
-}
-}, f)
-
 def _write_dummy_report(self, filename):
 from ooni.reporter import YAMLReporter
 from .test_reporter import test_details
@@ -65,10 +54,9 @@ class TestOONIReport(ConfigTestCase):
 mock_oonib_report_log_i.closed.return_value = defer.succeed(True)
 
 report_name = "dummy_report.yaml"
-self._create_reporting_yaml(report_name)
 self._write_dummy_report(report_name)
 from ooni.scripts import oonireport
-d = oonireport.upload(report_name)
+d = oonireport.upload(report_name, 
collector='httpo://thirteenchars123.onion')
 @d.addCallback
 def cb(result):
 mock_oonib_reporter_i.writeReportEntry.assert_called_with(
@@ -90,14 +78,13 @@ class TestOONIReport(ConfigTestCase):
 mock_oonib_report_log_i = mock_oonib_report_log.return_value
 mock_oonib_report_log_i.created.return_value = defer.succeed(True)
 mock_oonib_report_log_i.closed.return_value = defer.succeed(True)
-mock_oonib_report_log_i.reports_to_upload = [("dummy_report.yaml", 
None)]
+mock_oonib_report_log_i.get_to_upload.return_value = 
defer.succeed([("dummy_report.yaml", {'measurement_id': 'XX'})])
 
 report_name = "dummy_report.yaml"
-self._create_reporting_yaml(report_name)
 self._write_dummy_report(report_name)
 
 from ooni.scripts import oonireport
-d = oonireport.upload_all()
+d = oonireport.upload_all(collector='httpo://thirteenchars123.onion')
 @d.addCallback
 def cb(result):
 mock_oonib_reporter_i.writeReportEntry.assert_called_with(



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Implement CSRF protection based on double-submit token.

[art]

commit 84e7083fe0c0ad53286756395f07b7a9cbb10c18
Author: Arturo Filastò 
Date:   Wed Jul 27 18:00:34 2016 +0200

Implement CSRF protection based on double-submit token.
---
 ooni/deck.py  |  2 +-
 ooni/ui/web/server.py | 68 +++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 7976e5e..6844fda 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -492,7 +492,7 @@ class InputStore(object):
 def list(self):
 if self._cache_stale:
 self._update_cache()
-return self._cache
+return deepcopy(self._cache)
 
 def get(self, input_id):
 if self._cache_stale:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index f14f6b8..a6fd315 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -2,6 +2,9 @@ from __future__ import print_function
 
 import os
 import json
+import string
+from functools import wraps
+from random import SystemRandom
 
 from twisted.internet import defer, task, reactor
 from twisted.python import usage
@@ -26,11 +29,52 @@ def rpath(*path):
 context = os.path.abspath(os.path.dirname(__file__))
 return os.path.join(context, *path)
 
+
 class WebUIError(Exception):
 def __init__(self, code, message):
 self.code = code
 self.message = message
 
+
+def xsrf_protect(check=True):
+"""
+This is a decorator that implements double submit token CSRF protection.
+
+Basically we set a cookie and ensure that every request contains the
+same value inside of the cookie and the request header.
+
+It's based on the assumption that an attacker cannot read the cookie
+that is set by the server (since it would be violating the SOP) and hence
+is not possible to make a browser trigger requests that contain the
+cookie value inside of the requests it sends.
+
+If you wish to disable checking of the token set the value check to False.
+This will still lead to the cookie being set.
+
+This decorator needs to be applied after the decorator that registers
+the routes.
+"""
+def deco(f):
+
+@wraps(f)
+def wrapper(instance, request, *a, **kw):
+should_check = check and instance._enable_xsrf_protection
+token_cookie = request.getCookie(u'XSRF-TOKEN')
+token_header = request.getHeader(b"X-XSRF-TOKEN")
+if (token_cookie != instance._xsrf_token and
+instance._enable_xsrf_protection):
+request.addCookie(u'XSRF-TOKEN',
+  instance._xsrf_token)
+if should_check and token_cookie != token_header:
+raise WebUIError(404, "Invalid XSRF token")
+return f(instance, request, *a, **kw)
+
+return wrapper
+
+return deco
+
+
+
 class LongPoller(object):
 def __init__(self, timeout, _reactor=reactor):
 self.lock = defer.DeferredLock()
@@ -73,6 +117,7 @@ class WebUIAPI(object):
 # change happenned.
 _long_polling_timeout = 5
 _reactor = reactor
+_enable_xsrf_protection = True
 
 def __init__(self, config, director, _reactor=reactor):
 self.director = director
@@ -80,6 +125,12 @@ class WebUIAPI(object):
 self.measurement_path = FilePath(config.measurements_directory)
 self.decks_path = FilePath(config.decks_directory)
 
+# We use a double submit token to protect against XSRF
+rng = SystemRandom()
+token_space = string.letters+string.digits
+self._xsrf_token = ''.join([rng.choice(token_space)
+for _ in range(30)])
+
 self.status = {
 "software_version": ooniprobe_version,
 "software_name": "ooniprobe",
@@ -113,10 +164,12 @@ class WebUIAPI(object):
 self.status["country_code"] = probe_ip.geodata['countrycode']
 
 @app.handle_errors(NotFound)
+@xsrf_protect(check=False)
 def not_found(self, request, _):
 request.redirect('/client/')
 
 @app.handle_errors(WebUIError)
+@xsrf_protect(check=False)
 def web_ui_error(self, request, failure):
 error = failure.value
 request.setResponseCode(error.code)
@@ -132,6 +185,7 @@ class WebUIAPI(object):
 return json_string
 
 @app.route('/api/notify', methods=["GET"])
+@xsrf_protect(check=False)
 def api_notify(self, request):
 def got_director_event(event):
 return self.render_json({
@@ -143,10 +197,12 @@ class WebUIAPI(object):
 return d
 
 @app.route('/api/status', methods=["GET"])
+@xsrf_protect(check=False)
 def api_status(self, request):
 return self.render_json(self.status, request)
 
 @app.route('/api/status/update', methods=["GET"])
+@xsrf_protect(check=False)
 def api_status_update(self, request):
 def got_status_update(event):
 return 

[tor-commits] [ooni-probe/master] Add support for loglevels.

[art]

commit 5a6a483bd0e1f1443f170955085543ef11aac34f
Author: Arturo Filastò 
Date:   Wed Jul 27 23:47:45 2016 +0200

Add support for loglevels.

Small refactor of the logging facilities

Note: now all messages printed with print() are not output to stdout by 
default
and all messages that are not printed via the ooni logging infrastructure 
that
are not critical are also ignored.

This is useful to avoid cluttering the CLI with too many messages and also 
to
reduce the size of the logs.

This implements: #502
---
 ooni/nettest.py   |   8 +--
 ooni/settings.py  |  25 ++---
 ooni/ui/cli.py|  12 ++--
 ooni/utils/log.py | 160 +++---
 4 files changed, 133 insertions(+), 72 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 566c391..1978c68 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -148,7 +148,7 @@ def usageOptionsFactory(test_name, test_version):
 """
 Display the net_test version and exit.
 """
-print "{} version: {}".format(test_name, test_version)
+log.msg("{} version: {}".format(test_name, test_version))
 sys.exit(0)
 
 return UsageOptions
@@ -513,13 +513,13 @@ class NetTest(object):
 
 def doneNetTest(self, result):
 if self.summary:
-print "Summary for %s" % self.testDetails['test_name']
-print "" + "-"*len(self.testDetails['test_name'])
+log.msg("Summary for %s" % self.testDetails['test_name'])
+log.msg("" + "-"*len(self.testDetails['test_name']))
 for test_class in self.uniqueClasses():
 test_instance = test_class()
 test_instance.displaySummary(self.summary)
 if self.testDetails["report_id"]:
-print "Report ID: %s" % self.testDetails["report_id"]
+log.msg("Report ID: %s" % self.testDetails["report_id"])
 
 def doneReport(self, report_results):
 """
diff --git a/ooni/settings.py b/ooni/settings.py
index acb1895..194729a 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -9,7 +9,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from os.path import abspath, expanduser
 
 from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
-from ooni import geoip
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
@@ -101,8 +100,6 @@ class OConfig(object):
 self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
 
 self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler')
-if not os.path.exists(self.scheduler_directory):
-os.mkdir(self.scheduler_directory)
 
 if self.advanced.decks_dir:
 self.decks_directory = self.advanced.decks_dir
@@ -136,11 +133,25 @@ class OConfig(object):
 
 ooni_home = self.ooni_home
 if not os.path.isdir(ooni_home):
-print "Ooni home directory does not exist."
-print "Creating it in '%s'." % ooni_home
+log.msg("Ooni home directory does not exist")
+log.msg("Creating it in '%s'" % ooni_home)
 os.mkdir(ooni_home)
-os.mkdir(self.inputs_directory)
-os.mkdir(self.decks_directory)
+
+# also ensure the subdirectories exist
+sub_directories = [
+self.inputs_directory,
+self.decks_directory,
+self.scheduler_directory,
+self.measurements_directory,
+self.resources_directory
+]
+for path in sub_directories:
+try:
+os.mkdir(path)
+except OSError as exc:
+if exc.errno != 17:
+raise
+
 
 def _create_config_file(self):
 target_config_file = self.config_file
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 51ed3b9..2f5c467 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -339,9 +339,9 @@ def runWithDirector(global_options, 
create_input_store=True):
 director = Director()
 if global_options['list']:
 net_tests = [net_test for net_test in director.getNetTests().items()]
-print ""
-print "Installed nettests"
-print "=="
+log.msg("")
+log.msg("Installed nettests")
+log.msg("==")
 for net_test_id, net_test in net_tests:
 optList = []
 for name, details in net_test['arguments'].items():
@@ -359,9 +359,9 @@ def runWithDirector(global_options, 
create_input_store=True):
 '\n\n' +
 ''.join(usage.docMakeChunks(optList))
 )
-print desc
-print "Note: Third party tests require an external "\
-  "application to run properly."
+map(log.msg, desc.split("\n"))
+log.msg("Note: Third 

[tor-commits] [ooni-probe/master] Implement deck store

[art]

commit 97ed0fcbda310db18b2138f70024f08de5b523b9
Author: Arturo Filastò 
Date:   Fri Jul 29 22:50:45 2016 +0200

Implement deck store

* Write all runtime files to /var/lib/ooni

* Other various fixes to runtime paths

* Include the default deck and data files in distribution
---
 MANIFEST.in |  2 ++
 data/decks/web.yaml | 22 +
 ooni/agent/agent.py |  8 -
 ooni/agent/scheduler.py | 22 -
 ooni/deck/deck.py   | 15 +++--
 ooni/deck/store.py  | 35 +---
 ooni/scripts/ooniprobe_agent.py | 17 +++---
 ooni/settings.py| 72 -
 ooni/ui/web/server.py   |  8 +
 ooni/utils/log.py   |  2 ++
 10 files changed, 153 insertions(+), 50 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 692c73f..485e834 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,6 @@
+recursive-include ooni/ui/web/client *
 include README.rst ChangeLog.rst requirements.txt LICENSE
+recursive-include data/decks *
 include data/oonideckgen.1
 include data/ooniprobe.1
 include data/oonireport.1
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
new file mode 100644
index 000..a81b8f8
--- /dev/null
+++ b/data/decks/web.yaml
@@ -0,0 +1,22 @@
+---
+name: Web related ooniprobe tests
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+Request and the Web Connectivity test
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+  ooni:
+test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+test_name: web_connectivity
+file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index ff9a2dd..c2e7e26 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -1,7 +1,6 @@
 from twisted.application import service
 from ooni.director import Director
 from ooni.settings import config
-from ooni.utils import log
 
 from ooni.ui.web.web import WebUIService
 from ooni.agent.scheduler import SchedulerService
@@ -11,9 +10,6 @@ class AgentService(service.MultiService):
 service.MultiService.__init__(self)
 
 director = Director()
-config.set_paths()
-config.initialize_ooni_home()
-config.read_config_file()
 
 self.web_ui_service = WebUIService(director, web_ui_port)
 self.web_ui_service.setServiceParent(self)
@@ -24,9 +20,5 @@ class AgentService(service.MultiService):
 def startService(self):
 service.MultiService.startService(self)
 
-log.start()
-
 def stopService(self):
 service.MultiService.stopService(self)
-
-log.stop()
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 167a14a..43715e8 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -7,7 +7,7 @@ from twisted.python.filepath import FilePath
 from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
-from ooni.deck.store import input_store
+from ooni.deck.store import input_store, deck_store
 from ooni.settings import config
 from ooni.contrib import croniter
 from ooni.geoip import probe_ip
@@ -116,6 +116,25 @@ class DeleteOldReports(ScheduledTask):
 log.debug("Deleting old report {0}".format(measurement["id"]))
 measurement_path.child(measurement['id']).remove()
 
+
+class RunDecks(ScheduledTask):
+"""
+This will run the decks that have been configured on the system as the
+decks to run by default.
+"""
+schedule = '@daily'
+identifier = 'run-decks'
+
+def __init__(self, director, schedule=None):
+super(RunDecks, self).__init__(schedule)
+self.director = director
+
+@defer.inlineCallbacks
+def task(self):
+for deck_id, deck in deck_store.list():
+yield deck.setup()
+yield deck.run(self.director)
+
 class SendHeartBeat(ScheduledTask):
 """
 This task is used to send a heartbeat that the probe is still alive and
@@ -193,6 +212,7 @@ class SchedulerService(service.MultiService):
 self.schedule(UpdateInputsAndResources())
 self.schedule(UploadReports())
 self.schedule(DeleteOldReports())
+self.schedule(RunDecks(self.director))
 
 self._looping_call.start(self.interval)
 
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 868bfb8..1b2300a 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,5 +1,6 @@
 import os
 from copy import deepcopy
+from string import Template
 
 import yaml
 from twisted.internet import defer
@@ -10,7 +11,6 @@ from ooni.backend_client 

[tor-commits] [ooni-probe/master] Add commands for managing the lifecycle of the ooniprobe-agent

[art]

commit fe64930f836dccf76a87b9e9bcfe655bada49f90
Author: Arturo Filastò 
Date:   Fri Jul 29 16:47:32 2016 +0200

Add commands for managing the lifecycle of the ooniprobe-agent

* Add command to start the web-ui from ooniprobe
* Improvements to the measurements API
* Add klein dependency
---
 ooni/agent/agent.py |   4 +-
 ooni/measurements.py|  93 ++--
 ooni/reporter.py|  13 +++-
 ooni/scripts/ooniprobe.py   |  15 -
 ooni/scripts/ooniprobe_agent.py | 130 +++-
 ooni/ui/cli.py  |   5 +-
 ooni/ui/web/client/index.html   |   2 +-
 ooni/ui/web/server.py   |  53 +---
 requirements.txt|   1 +
 9 files changed, 247 insertions(+), 69 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index f5322ed..ff9a2dd 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -7,7 +7,7 @@ from ooni.ui.web.web import WebUIService
 from ooni.agent.scheduler import SchedulerService
 
 class AgentService(service.MultiService):
-def __init__(self):
+def __init__(self, web_ui_port):
 service.MultiService.__init__(self)
 
 director = Director()
@@ -15,7 +15,7 @@ class AgentService(service.MultiService):
 config.initialize_ooni_home()
 config.read_config_file()
 
-self.web_ui_service = WebUIService(director)
+self.web_ui_service = WebUIService(director, web_ui_port)
 self.web_ui_service.setServiceParent(self)
 
 self.scheduler_service = SchedulerService(director)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index f830a99..50efd87 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -1,10 +1,14 @@
+import os
 import json
+import signal
+
 from twisted.python.filepath import FilePath
 from ooni.settings import config
 
 class Process():
 supported_tests = [
-"web_connectivity"
+"web_connectivity",
+"http_requests"
 ]
 @staticmethod
 def web_connectivity(entry):
@@ -15,6 +19,22 @@ class Process():
 result['url'] = entry['input']
 return result
 
+@staticmethod
+def http_requests(entry):
+result = {}
+test_keys = entry['test_keys']
+anomaly = (
+test_keys['body_length_match'] and
+test_keys['headers_match'] and
+(
+test_keys['control_failure'] !=
+test_keys['experiment_failure']
+)
+)
+result['anomaly'] = anomaly
+result['url'] = entry['input']
+return result
+
 def generate_summary(input_file, output_file):
 results = {}
 with open(input_file) as in_file:
@@ -34,29 +54,64 @@ def generate_summary(input_file, output_file):
 with open(output_file, "w") as fw:
 json.dump(results, fw)
 
+class MeasurementNotFound(Exception):
+pass
+
+def get_measurement(measurement_id):
+measurement_path = FilePath(config.measurements_directory)
+measurement = measurement_path.child(measurement_id)
+if not measurement.exists():
+raise MeasurementNotFound
+
+running = False
+completed = True
+keep = False
+if measurement.child("measurement.njson.progress").exists():
+completed = False
+# XXX this is done quite often around the code, probably should
+# be moved into some utility function.
+pid = measurement.child("running.pid").open("r").read()
+pid = int(pid)
+try:
+os.kill(pid, signal.SIG_DFL)
+running = True
+except OSError:
+pass
+
+if measurement.child("keep").exists():
+keep = True
+test_start_time, country_code, asn, test_name = \
+measurement_id.split("-")[:4]
+return {
+"test_name": test_name,
+"country_code": country_code,
+"asn": asn,
+"test_start_time": test_start_time,
+"id": measurement_id,
+"completed": completed,
+"keep": keep,
+"running": running
+}
+
+
+def get_summary(measurement_id):
+measurement_path = FilePath(config.measurements_directory)
+measurement = measurement_path.child(measurement_id)
+summary = measurement.child("summary.json")
+if not summary.exists():
+generate_summary(
+measurement.child("measurement.njson").path,
+summary.path
+)
+
+with summary.open("r") as f:
+return json.load(f)
 
 def list_measurements():
 measurements = []
 measurement_path = FilePath(config.measurements_directory)
 for measurement_id in measurement_path.listdir():
-measurement = measurement_path.child(measurement_id)
-completed = True
-keep = False
-if measurement.child("measurement.njson.progress").exists():
-completed = False
-if measurement.child("keep").exists():
-   

[tor-commits] [ooni-probe/master] Add support for listing enabled and disabled decks

[art]

commit 7c35e65485418d1dcb682bad1cf4eb62ef318e81
Author: Arturo Filastò 
Date:   Sat Jul 30 16:27:20 2016 +0200

Add support for listing enabled and disabled decks

* Fix various bugs
---
 ooni/agent/scheduler.py   |  2 +-
 ooni/deck/store.py| 40 +
 ooni/director.py  |  5 +--
 ooni/measurements.py  | 11 --
 ooni/settings.py  |  9 +++--
 ooni/ui/web/client/index.html |  2 +-
 ooni/ui/web/server.py | 81 +--
 7 files changed, 117 insertions(+), 33 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 43715e8..7a77afb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -131,7 +131,7 @@ class RunDecks(ScheduledTask):
 
 @defer.inlineCallbacks
 def task(self):
-for deck_id, deck in deck_store.list():
+for deck_id, deck in deck_store.list_enabled():
 yield deck.setup()
 yield deck.run(self.director)
 
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 7c90204..695f97d 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -9,7 +9,6 @@ from ooni.deck.deck import NGDeck
 from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
 from ooni.settings import config
-from ooni.utils import log
 
 class InputNotFound(Exception):
 pass
@@ -121,25 +120,54 @@ class InputStore(object):
 
 class DeckStore(object):
 def __init__(self):
-self.path = FilePath(config.decks_directory)
+self.enabled_directory = FilePath(config.decks_enabled_directory)
+self.available_directory = FilePath(config.decks_available_directory)
 self._cache = {}
 self._cache_stale = True
 
-def list(self):
-decks = []
+def _list(self):
 if self._cache_stale:
 self._update_cache()
 for deck_id, deck in self._cache.iteritems():
+yield (deck_id, deck)
+
+def list(self):
+decks = []
+for deck_id, deck in self._list():
 decks.append((deck_id, deck))
 return decks
 
+def list_enabled(self):
+decks = []
+for deck_id, deck in self._list():
+if self.is_enabled(deck_id):
+continue
+decks.append((deck_id, deck))
+return decks
+
+def is_enabled(self, deck_id):
+return self.enabled_directory.child(deck_id + '.yaml').exists()
+
+def enable(self, deck_id):
+deck_path = self.available_directory.child(deck_id + '.yaml')
+if not deck_path.exists():
+raise DeckNotFound(deck_id)
+deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
+deck_enabled_path.linkTo(deck_path)
+
+def disable(self, deck_id):
+deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
+if not deck_enabled_path.exists():
+raise DeckNotFound(deck_id)
+deck_enabled_path.remove()
+
 def _update_cache(self):
-for deck_path in self.path.listdir():
+for deck_path in self.available_directory.listdir():
 if not deck_path.endswith('.yaml'):
 continue
 deck_id = deck_path[:-1*len('.yaml')]
 deck = NGDeck(
-deck_path=self.path.child(deck_path).path
+deck_path=self.available_directory.child(deck_path).path
 )
 self._cache[deck_id] = deck
 
diff --git a/ooni/director.py b/ooni/director.py
index 464a203..304a14a 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -9,7 +9,7 @@ from ooni.utils import log, generate_filename
 from ooni.nettest import NetTest, getNetTestInformation
 from ooni.settings import config
 from ooni.nettest import normalizeTestName
-from ooni.deck.store import InputStore
+from ooni.deck.store import input_store, deck_store
 from ooni.geoip import probe_ip
 
 from ooni.agent.scheduler import run_system_tasks
@@ -96,7 +96,8 @@ class Director(object):
 self.allTestsDone = defer.Deferred()
 self.sniffers = {}
 
-self.input_store = InputStore()
+self.input_store = input_store
+self.deck_store = deck_store
 
 self._reset_director_state()
 self._reset_tor_state()
diff --git a/ooni/measurements.py b/ooni/measurements.py
index 50efd87..cdf6b14 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -5,6 +5,9 @@ import signal
 from twisted.python.filepath import FilePath
 from ooni.settings import config
 
+class MeasurementInProgress(Exception):
+pass
+
 class Process():
 supported_tests = [
 "web_connectivity",
@@ -66,7 +69,7 @@ def get_measurement(measurement_id):
 running = False
 completed = True
 keep = False
-if measurement.child("measurement.njson.progress").exists():
+if measurement.child("measurements.njson.progress").exists():
 completed = 

[tor-commits] [ooni-probe/master] Implement tasks and add new APIs

[art]

commit c71375eed5db6dd1e11881c7045c2392c68aa42c
Author: Arturo Filastò 
Date:   Fri Jul 29 00:20:45 2016 +0200

Implement tasks and add new APIs

* Add new API endpoints for deleting and keeping reports

* Implement tasks for uploading not submitted reports

* Implement tasks for cleaning up old reports
---
 ooni/agent/agent.py   | 11 +++
 ooni/agent/scheduler.py   | 69 ---
 ooni/deck/deck.py |  2 +-
 ooni/measurements.py  | 67 +
 ooni/results.py   | 40 -
 ooni/scripts/oonireport.py|  8 +
 ooni/ui/web/client/index.html |  2 +-
 ooni/ui/web/server.py | 40 +++--
 ooni/utils/__init__.py|  6 ++--
 9 files changed, 188 insertions(+), 57 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index a1394f0..f5322ed 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -1,6 +1,7 @@
 from twisted.application import service
 from ooni.director import Director
 from ooni.settings import config
+from ooni.utils import log
 
 from ooni.ui.web.web import WebUIService
 from ooni.agent.scheduler import SchedulerService
@@ -19,3 +20,13 @@ class AgentService(service.MultiService):
 
 self.scheduler_service = SchedulerService(director)
 self.scheduler_service.setServiceParent(self)
+
+def startService(self):
+service.MultiService.startService(self)
+
+log.start()
+
+def stopService(self):
+service.MultiService.stopService(self)
+
+log.stop()
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 1288d6c..167a14a 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -4,12 +4,17 @@ from twisted.application import service
 from twisted.internet import task, defer
 from twisted.python.filepath import FilePath
 
+from ooni.scripts import oonireport
 from ooni import resources
-from ooni.utils import log
+from ooni.utils import log, SHORT_DATE
 from ooni.deck.store import input_store
 from ooni.settings import config
 from ooni.contrib import croniter
 from ooni.geoip import probe_ip
+from ooni.measurements import list_measurements
+
+class DidNotRun(Exception):
+pass
 
 class ScheduledTask(object):
 _time_format = "%Y-%m-%dT%H:%M:%SZ"
@@ -58,7 +63,7 @@ class ScheduledTask(object):
 yield self._last_run_lock.deferUntilLocked()
 if not self.should_run:
 self._last_run_lock.unlock()
-defer.returnValue(None)
+raise DidNotRun
 try:
 yield self.task()
 self._update_last_run()
@@ -68,7 +73,7 @@ class ScheduledTask(object):
 self._last_run_lock.unlock()
 
 class UpdateInputsAndResources(ScheduledTask):
-identifier = "ooni-update-inputs"
+identifier = "update-inputs"
 schedule = "@daily"
 
 @defer.inlineCallbacks
@@ -78,13 +83,50 @@ class UpdateInputsAndResources(ScheduledTask):
 yield resources.check_for_update(probe_ip.geodata['countrycode'])
 yield input_store.update(probe_ip.geodata['countrycode'])
 
-class CleanupInProgressReports(ScheduledTask):
-identifier = 'ooni-cleanup-reports'
+
+class UploadReports(ScheduledTask):
+"""
+This task is used to submit to the collector reports that have not been
+submitted and those that have been partially uploaded.
+"""
+identifier = 'upload-reports'
+schedule = '@hourly'
+
+@defer.inlineCallbacks
+def task(self):
+yield oonireport.upload_all(upload_incomplete=True)
+
+
+class DeleteOldReports(ScheduledTask):
+"""
+This task is used to delete reports that are older than a week.
+"""
+identifier = 'delete-old-reports'
 schedule = '@daily'
 
-class UploadMissingReports(ScheduledTask):
-identifier = 'ooni-cleanup-reports'
-schedule = '@weekly'
+def task(self):
+measurement_path = FilePath(config.measurements_directory)
+for measurement in list_measurements():
+if measurement['keep'] is True:
+continue
+delta = datetime.utcnow() - \
+datetime.strptime(measurement['test_start_time'],
+  SHORT_DATE)
+if delta.days >= 7:
+log.debug("Deleting old report {0}".format(measurement["id"]))
+measurement_path.child(measurement['id']).remove()
+
+class SendHeartBeat(ScheduledTask):
+"""
+This task is used to send a heartbeat that the probe is still alive and
+well.
+"""
+identifier = 'send-heartbeat'
+schedule = '@hourly'
+
+def task(self):
+# XXX implement this
+pass
 
 # Order mattters
 SYSTEM_TASKS = [
@@ -122,8 +164,12 @@ class SchedulerService(service.MultiService):
 def schedule(self, task):
 self._scheduled_tasks.append(task)
 
+def 

[tor-commits] [ooni-probe/master] Fix path to the web ui root

[art]

commit 9768924e8cd53c31f751fa0b605a6f6c502b23be
Author: Arturo Filastò 
Date:   Fri Jul 29 23:54:19 2016 +0200

Fix path to the web ui root
---
 ooni/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index 5762919..3d33601 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -131,7 +131,7 @@ class OConfig(object):
 
 def set_paths(self):
 self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
-self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client')
+self.web_ui_directory = os.path.join(get_ooni_root(), 'ui', 'web', 
'client')
 
 self.inputs_directory = os.path.join(self.running_path, 'inputs')
 self.scheduler_directory = os.path.join(self.running_path, 'scheduler')



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Edit ooniprobe configuration file

[art]

commit 921a561062c6fc9cba375c61e4638ac33100acbe
Author: Arturo Filastò 
Date:   Fri Jul 29 23:56:20 2016 +0200

Edit ooniprobe configuration file
---
 data/ooniprobe.conf.sample | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index cde4c00..f4c0812 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -5,6 +5,7 @@
 basic:
 # Where OONIProbe should be writing it's log file
 logfile: /var/log/ooniprobe.log
+loglevel: WARNING
 privacy:
 # Should we include the IP address of the probe in the report?
 includeip: false
@@ -44,10 +45,6 @@ advanced:
 reporting_retries: 5
 # How many reports to perform concurrently
 reporting_concurrency: 7
-oonid_api_port: 8042
-report_log_file: null
-inputs_dir: null
-decks_dir: null
 # If we should support communicating to plaintext backends (via HTTP)
 insecure_backend: false
 # The preferred backend type, can be one of onion, https or cloudfront



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] Forward ports from the vagrant file

[art]

commit cd18ae75cb2e63e05649adc166dd5c91d6c3fde1
Author: Arturo Filastò 
Date:   Sat Jul 30 00:36:10 2016 +0200

Forward ports from the vagrant file
---
 Vagrantfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Vagrantfile b/Vagrantfile
index 59b2751..cf73244 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -54,6 +54,7 @@ Vagrant.configure("2") do |config|
   config.vm.box = "debian/contrib-jessie64"
 
   config.vm.define "probe" do |probe|
+probe.vm.network "forwarded_port", guest: 8842, host: 8042
 probe.vm.synced_folder ".", "/data/ooni-probe"
 probe.vm.provision :shell, :inline => $setup_ooniprobe
   end



___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [ooni-probe/master] The report log now is used only for measurements that are written to the ~/.ooni/measurements directory.

[art]

commit 0362ad88fbe2945f5311b3db5b67031a0e8b218d
Author: Arturo Filastò 
Date:   Thu Jul 28 17:32:12 2016 +0200

The report log now is used only for measurements that are written to the 
~/.ooni/measurements directory.

* Move tor related functions into utils onion.
---
 ooni/deck/deck.py   |   4 +-
 ooni/director.py|  55 +---
 ooni/reporter.py| 296 +++-
 ooni/scripts/oonireport.py  | 128 ++-
 ooni/tests/test_reporter.py |  87 ++---
 ooni/utils/log.py   |   4 +-
 ooni/utils/onion.py |  49 
 7 files changed, 352 insertions(+), 271 deletions(-)

diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index b11e174..c9b3fc5 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -218,6 +218,7 @@ class NGDeck(object):
 net_test_loader = task.ooni["net_test_loader"]
 test_details = task.ooni["test_details"]
 
+measurement_id = None
 report_filename = task.output_path
 if not task.output_path:
 measurement_id = task.id
@@ -235,7 +236,8 @@ class NGDeck(object):
 net_test_loader,
 report_filename,
 collector_client=net_test_loader.collector,
-test_details=test_details
+test_details=test_details,
+measurement_id=measurement_id
 )
 d.addCallback(self._measurement_completed, task)
 d.addErrback(self._measurement_failed, task)
diff --git a/ooni/director.py b/ooni/director.py
index e3df907..464a203 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,4 +1,3 @@
-import pwd
 import os
 
 from twisted.internet import defer
@@ -7,7 +6,6 @@ from twisted.python.failure import Failure
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
 from ooni.utils import log, generate_filename
-from ooni.utils.net import randomFreePort
 from ooni.nettest import NetTest, getNetTestInformation
 from ooni.settings import config
 from ooni.nettest import normalizeTestName
@@ -15,7 +13,7 @@ from ooni.deck.store import InputStore
 from ooni.geoip import probe_ip
 
 from ooni.agent.scheduler import run_system_tasks
-from ooni.utils.onion import start_tor, connect_to_control_port
+from ooni.utils.onion import start_tor, connect_to_control_port, get_tor_config
 
 class DirectorEvent(object):
 def __init__(self, type="update", message=""):
@@ -299,7 +297,7 @@ class Director(object):
 @defer.inlineCallbacks
 def start_net_test_loader(self, net_test_loader, report_filename,
   collector_client=None, no_yamloo=False,
-  test_details=None):
+  test_details=None, measurement_id=None):
 """
 Create the Report for the NetTest and start the report NetTest.
 
@@ -319,7 +317,8 @@ class Director(object):
 report = Report(test_details, report_filename,
 self.reportEntryManager,
 collector_client,
-no_yamloo)
+no_yamloo,
+measurement_id)
 
 yield report.open()
 net_test = NetTest(test_cases, test_details, report)
@@ -392,50 +391,8 @@ class Director(object):
 raise exc
 
 if config.advanced.start_tor and config.tor_state is None:
-tor_config = TorConfig()
-if config.tor.control_port is None:
-config.tor.control_port = int(randomFreePort())
-if config.tor.socks_port is None:
-config.tor.socks_port = int(randomFreePort())
-
-tor_config.ControlPort = config.tor.control_port
-tor_config.SocksPort = config.tor.socks_port
-
-if config.tor.data_dir:
-data_dir = os.path.expanduser(config.tor.data_dir)
-
-if not os.path.exists(data_dir):
-log.debug("%s does not exist. Creating it." % data_dir)
-os.makedirs(data_dir)
-tor_config.DataDirectory = data_dir
-
-if config.tor.bridges:
-tor_config.UseBridges = 1
-if config.advanced.obfsproxy_binary:
-tor_config.ClientTransportPlugin = (
-'obfs2,obfs3 exec %s managed' %
-config.advanced.obfsproxy_binary
-)
-bridges = []
-with open(config.tor.bridges) as f:
-for bridge in f:
-if 'obfs' in bridge:
-if config.advanced.obfsproxy_binary:
-bridges.append(bridge.strip())
-else:
-bridges.append(bridge.strip())
-tor_config.Bridge = bridges
-
-if config.tor.torrc:
-for