Merge branch 'dev' into cc20

edge.c

@@ -180,6 +180,8 @@ static void help() {
 #ifdef HAVE_OPENSSL_1_1
   printf("-A4                      | Use ChaCha20 for payload encryption. Requires a key.\n");
 #endif
+  printf("-z                       | Enable lzo1x compression for outgoing data packets\n");
+  printf("                         | (default=disabled).\n");
   printf("-E                       | Accept multicast MAC addresses (default=drop).\n");
   printf("-S                       | Do not connect P2P. Always use the supernode.\n");
 #ifdef __linux__
@@ -334,6 +336,12 @@ static int setOption(int optkey, char *optargument, n2n_priv_config_t *ec, n2n_e
       break;
     }
 
+  case 'z':
+    {
+      conf->compression = N2N_COMPRESSION_ID_LZO;
+      break;
+    }
+
   case 'l': /* supernode-list */
     if(optargument) {
       if(edge_conf_add_supernode(conf, optargument) != 0) {
@@ -441,8 +449,7 @@ static int loadFromCLI(int argc, char *argv[], n2n_edge_conf_t *conf, n2n_priv_c
   u_char c;
 
   while((c = getopt_long(argc, argv,
-			 "k:a:bc:Eu:g:m:M:s:d:l:p:fvhrt:i:SDL:z"
-			 "A::"
+			 "k:a:bc:Eu:g:m:M:s:d:l:p:fvhrt:i:SDL:zA::"
 #ifdef __linux__
 			 "T:"
 #endif

edge_utils.c

@@ -47,6 +47,10 @@
 #define IP4_MIN_SIZE  20
 #define UDP_SIZE      8
 
+/* heap allocation for compression as per lzo example doc */
+#define HEAP_ALLOC(var,size) lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
+static HEAP_ALLOC(wrkmem, LZO1X_1_MEM_COMPRESS);
+
 /* ************************************** */
 
 static const char * supernode_ip(const n2n_edge_t * eee);
@@ -219,12 +223,10 @@ n2n_edge_t* edge_init(const tuntap_dev *dev, const n2n_edge_conf_t *conf, int *r
   eee->pending_peers  = NULL;
   eee->sup_attempts = N2N_EDGE_SUP_ATTEMPTS;
 
-#ifdef NOT_USED
   if(lzo_init() != LZO_E_OK) {
     traceEvent(TRACE_ERROR, "LZO compression error");
     goto edge_init_error;
   }
-#endif
 
   for(i=0; i<conf->sn_num; ++i)
     traceEvent(TRACE_NORMAL, "supernode %u => %s\n", i, (conf->sn_ip_array[i]));
@@ -950,6 +952,15 @@ static int handle_PACKET(n2n_edge_t * eee,
     n2n_transform_t rx_transop_id;
 
     rx_transop_id = (n2n_transform_t)pkt->transform;
+    /* optional compression is encoded in uppermost bit of transform field.
+     * this is an intermediate solution to maintain compatibility until some
+     * upcoming major release (3.0?) brings up changes in packet structure anyway
+     * in the course of which a dedicated compression field could be spent.
+     * REVISIT then. */
+    uint16_t rx_compression_id;
+
+    rx_compression_id = (uint16_t)rx_transop_id >> (8*sizeof((uint16_t)rx_transop_id)-N2N_COMPRESSION_ID_BITLEN);
+    rx_transop_id &= (1 << (8*sizeof((uint16_t)rx_transop_id)-N2N_COMPRESSION_ID_BITLEN)) -1;
 
     if(rx_transop_id == eee->conf.transop_id) {
         uint8_t is_multicast;
@@ -959,6 +970,28 @@ static int handle_PACKET(n2n_edge_t * eee,
 						    eth_payload, N2N_PKT_BUF_SIZE,
 						    payload, psize, pkt->srcMac);
 	++(eee->transop.rx_cnt); /* stats */
+
+        /* decompress if necessary */
+        uint8_t * deflation_buffer = 0;
+        uint32_t deflated_len;
+        switch (rx_compression_id) {
+          case N2N_COMPRESSION_ID_LZO:
+	    deflation_buffer = malloc (N2N_PKT_BUF_SIZE);
+	    lzo1x_decompress (eth_payload, eth_size, deflation_buffer, (lzo_uint*)&deflated_len, NULL);
+            break;
+
+          default:
+            break;
+        }
+
+        if (rx_compression_id) {
+          traceEvent (TRACE_DEBUG, "payload decompression [id: %u]: deflated %u bytes to %u bytes",
+                                   rx_compression_id, eth_size, (int)deflated_len);
+          memcpy(eth_payload ,deflation_buffer, deflated_len );
+          eth_size = deflated_len;
+          free (deflation_buffer);
+        }
+
 	is_multicast = (is_ip6_discovery(eth_payload, eth_size) || is_ethMulticast(eth_payload, eth_size));
 
 	if(eee->conf.drop_multicast && is_multicast) {
@@ -1316,6 +1349,41 @@ static void send_packet2net(n2n_edge_t * eee,
   pkt.sock.family=0; /* do not encode sock */
   pkt.transform = tx_transop_idx;
 
+  // compression needs to be tried before encode_PACKET is called for compression indication gets encoded there
+  pkt.compression = N2N_COMPRESSION_ID_NONE;
+  if (eee->conf.compression) {
+    uint8_t * compression_buffer;
+    uint32_t  compression_len;
+    switch (eee->conf.compression) {
+        case N2N_COMPRESSION_ID_LZO:
+          compression_buffer = malloc (len + len / 16 + 64 + 3);
+          if (lzo1x_1_compress(tap_pkt, len, compression_buffer, (lzo_uint*)&compression_len, wrkmem) == LZO_E_OK) {
+            if (compression_len < len) {
+              pkt.compression = N2N_COMPRESSION_ID_LZO;
+            }
+          }
+          break;
+
+        default:
+          break;
+    }
+
+    if (pkt.compression) {
+      traceEvent (TRACE_DEBUG, "payload compression [id: %u]: compressed %u bytes to %u bytes\n",
+                               pkt.compression, len, compression_len);
+
+      memcpy (tap_pkt, compression_buffer, compression_len);
+      len = compression_len;
+      free (compression_buffer);
+    }
+  }
+  /* optional compression is encoded in uppermost bits of transform field.
+   * this is an intermediate solution to maintain compatibility until some
+   * upcoming major release (3.0?) brings up changes in packet structure anyway
+   * in the course of which a dedicated compression field could be spent.
+   * REVISIT then. */
+  pkt.transform = pkt.transform | (pkt.compression << (8*sizeof(pkt.transform)-N2N_COMPRESSION_ID_BITLEN));
+
   idx=0;
   encode_PACKET(pktbuf, &idx, &cmn, &pkt);
 
@@ -1922,6 +1990,7 @@ void edge_init_conf_defaults(n2n_edge_conf_t *conf) {
   conf->local_port = 0 /* any port */;
   conf->mgmt_port = N2N_EDGE_MGMT_PORT; /* 5644 by default */
   conf->transop_id = N2N_TRANSFORM_ID_NULL;
+  conf->compression = N2N_COMPRESSION_ID_NONE;
   conf->drop_multicast = 1;
   conf->allow_p2p = 1;
   conf->disable_pmtu_discovery = 1;

n2n.h

@@ -162,11 +162,16 @@ typedef struct tuntap_dev {
 #define MSG_TYPE_PEER_INFO              9
 #define MSG_TYPE_QUERY_PEER            10
 
-/* Set N2N_COMPRESSION_ENABLED to 0 to disable lzo1x compression of ethernet
- * frames. Doing this will break compatibility with the standard n2n packet
- * format so do it only for experimentation. All edges must be built with the
- * same value if they are to understand each other. */
-#define N2N_COMPRESSION_ENABLED 1
+/* N2N compression indicators. */
+/* Compression is disabled by default for outgoing packets if no cli
+ * option is given. All edges are built with decompression support so
+ * they are able to understand each other. */
+#define N2N_COMPRESSION_ID_NONE		0	/* default, see edge_init_conf_defaults(...) in edge_utils.c */
+#define N2N_COMPRESSION_ID_LZO		1	/* set if '-z' cli option is present, see setOption(...) in edge.c */
+
+#define N2N_COMPRESSION_ID_BITLEN	3	/* number of bits used for encoding compression id in the uppermost
+				 	           bits of transform_id; will be obsolete as soon as compression gets
+						   its own field in the packet. REVISIT then. */
 
 #define DEFAULT_MTU   1290
 
@@ -210,6 +215,7 @@ typedef struct n2n_edge_conf {
   n2n_sn_name_t       sn_ip_array[N2N_EDGE_NUM_SUPERNODES];
   n2n_community_t     community_name;         /**< The community. 16 full octets. */
   n2n_transform_t     transop_id;             /**< The transop to use. */
+  uint16_t	      compression;	      /**< Compress outgoing data packets before encryption */
   uint8_t             dyn_ip_mode;            /**< Interface IP address is dynamically allocated, eg. DHCP. */
   uint8_t             allow_routing;          /**< Accept packet no to interface address. */
   uint8_t             drop_multicast;         /**< Multicast ethernet addresses. */

n2n_wire.h

@@ -137,6 +137,7 @@ typedef struct n2n_PACKET
     n2n_mac_t           dstMac;
     n2n_sock_t          sock;
     uint16_t            transform;
+    uint16_t		compression;
 } n2n_PACKET_t;
 
 /* Linked with n2n_register_super in n2n_pc_t. Only from edge to supernode. */

transform_aes.c

@@ -35,7 +35,9 @@
 
 /* AES plaintext preamble */
 #define TRANSOP_AES_VER_SIZE     1       /* Support minor variants in encoding in one module. */
-#define TRANSOP_AES_IV_SEED_SIZE 8	/* size of transmitted random part of IV in bytes; leave it set to 8 for now */
+#define TRANSOP_AES_IV_SEED_SIZE 8    /* size of transmitted random part of IV in bytes; could range
+				       * from 0=lowest security (constant IV)  to  16=higest security
+				       * (fully random IV); default=8 */
 #define TRANSOP_AES_IV_PADDING_SIZE (N2N_AES_IVEC_SIZE - TRANSOP_AES_IV_SEED_SIZE)
 #define TRANSOP_AES_IV_KEY_BYTES (AES128_KEY_BYTES) /* use AES128 for IV encryption */
 #define TRANSOP_AES_PREAMBLE_SIZE (TRANSOP_AES_VER_SIZE + TRANSOP_AES_IV_SEED_SIZE)
@@ -43,7 +45,7 @@
 typedef unsigned char n2n_aes_ivec_t[N2N_AES_IVEC_SIZE];
 
 typedef struct transop_aes {
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
   EVP_CIPHER_CTX      *enc_ctx;	      /* openssl's reusable evp_* encryption context */
   EVP_CIPHER_CTX      *dec_ctx;	      /* openssl's reusable evp_* decryption context */
   const EVP_CIPHER    *cipher;	      /* cipher to use: e.g. EVP_aes_128_cbc */
@@ -61,7 +63,7 @@ typedef struct transop_aes {
 static int transop_deinit_aes(n2n_trans_op_t *arg) {
   transop_aes_t *priv = (transop_aes_t *)arg->priv;
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
   EVP_CIPHER_CTX_free(priv->enc_ctx);
   EVP_CIPHER_CTX_free(priv->dec_ctx);
 #endif
@@ -74,10 +76,10 @@ static int transop_deinit_aes(n2n_trans_op_t *arg) {
 
 /* ****************************************************** */
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
 /* get any erorr message out of openssl
    taken from https://en.wikibooks.org/wiki/OpenSSL/Error_handling */
-char *openssl_err_as_string (void) {
+static char *openssl_err_as_string (void) {
   BIO *bio = BIO_new (BIO_s_mem ());
   ERR_print_errors (bio);
   char *buf = NULL;
@@ -94,12 +96,35 @@ char *openssl_err_as_string (void) {
 
 /* ****************************************************** */
 
-static void set_aes_cbc_iv(transop_aes_t *priv, n2n_aes_ivec_t ivec, uint64_t iv_seed) {
+/* convert a given number of bytes from memory to hex string; taken (and modified) from
+   https://stackoverflow.com/questions/6357031/how-do-you-convert-a-byte-array-to-a-hexadecimal-string-in-c */
+const char* to_hex(unsigned char * in, size_t insz, char * out, size_t outsz)
+{
+  unsigned char * pin = in;
+  const char * hex = "0123456789abcdef";
+  char * pout = out;
+  for(; pin < in+insz; pout +=2, pin++){
+    pout[0] = hex[(*pin>>4) & 0xF];
+    pout[1] = hex[ *pin     & 0xF];
+    if (pout + 2 - out > outsz){
+      /* Better to truncate output string than overflow buffer */
+      /* it would be still better to either return a status */
+      /* or ensure the target buffer is large enough and it never happen */
+      break;
+      }
+    }
+    pout[2] = 0;
+    return out;
+}
+
+/* ****************************************************** */
+
+static void set_aes_cbc_iv(transop_aes_t *priv, n2n_aes_ivec_t ivec, uint8_t * iv_seed) {
   uint8_t iv_full[N2N_AES_IVEC_SIZE];
 
   /* Extend the seed to full block size with padding value */
   memcpy(iv_full, priv->iv_pad_val, TRANSOP_AES_IV_PADDING_SIZE);
-  memcpy(iv_full + TRANSOP_AES_IV_PADDING_SIZE, &iv_seed, TRANSOP_AES_IV_SEED_SIZE);
+  memcpy(iv_full + TRANSOP_AES_IV_PADDING_SIZE, iv_seed, TRANSOP_AES_IV_SEED_SIZE);
 
   /* Encrypt the IV with secret key to make it unpredictable.
    * As discussed in https://github.com/ntop/n2n/issues/72, it's important to
@@ -115,7 +140,7 @@ static void set_aes_cbc_iv(transop_aes_t *priv, n2n_aes_ivec_t ivec, uint64_t iv
 /** The aes packet format consists of:
  *
  *  - a 8-bit aes encoding version in clear text
- *  - a 64-bit random IV seed
+ *  - a TRANSOP_AES_IV_SEED_SIZE-sized [bytes] random IV seed
  *  - encrypted payload.
  *
  *  [V|II|DDDDDDDDDDDDDDDDDDDDD]
@@ -135,7 +160,7 @@ static int transop_encode_aes(n2n_trans_op_t * arg,
     if((in_len + TRANSOP_AES_PREAMBLE_SIZE) <= out_len) {
       int len=-1;
       size_t idx=0;
-      uint64_t iv_seed = 0;
+      uint8_t iv_seed[TRANSOP_AES_IV_SEED_SIZE];
       uint8_t padding = 0;
       n2n_aes_ivec_t enc_ivec = {0};
 
@@ -144,12 +169,20 @@ static int transop_encode_aes(n2n_trans_op_t * arg,
       /* Encode the aes format version. */
       encode_uint8(outbuf, &idx, N2N_AES_TRANSFORM_VERSION);
 
-      /* Generate and encode the IV seed.
-       * Using two calls to rand() because RAND_MAX is usually < 64bit
-       * (e.g. linux) and sometimes < 32bit (e.g. Windows).
-       */
-      iv_seed = ((((uint64_t)rand() & 0xFFFFFFFF)) << 32) | rand();
-      encode_buf(outbuf, &idx, &iv_seed, TRANSOP_AES_IV_SEED_SIZE);
+      /* Generate and encode the IV seed using as many calls to rand() as neccessary.
+       * Note: ( N2N_AES_IV_SEED_SIZE % sizeof(rand_value) ) not neccessarily equals 0. */
+      uint32_t rand_value;
+      int8_t i;
+      for (i = TRANSOP_AES_IV_SEED_SIZE; i >= sizeof(rand_value); i -= sizeof(rand_value)) {
+        rand_value = rand(); // CONCERN: rand() is not consideren cryptographicly secure, REPLACE later
+        memcpy(iv_seed + TRANSOP_AES_IV_SEED_SIZE - i, &rand_value, sizeof(rand_value));
+      }
+      /* Are there bytes left to fill? */
+      if (i != 0) {
+        rand_value = rand(); // CONCERN: rand() is not consideren cryptographicly secure, REPLACE later
+        memcpy(iv_seed, &rand_value, i);
+      }
+      encode_buf(outbuf, &idx, iv_seed, TRANSOP_AES_IV_SEED_SIZE);
 
       /* Encrypt the assembly contents and write the ciphertext after the iv seed. */
       /* len is set to the length of the cipher plain text to be encrpyted 
@@ -164,11 +197,13 @@ static int transop_encode_aes(n2n_trans_op_t * arg,
       len2 = ((len / AES_BLOCK_SIZE) + 1) * AES_BLOCK_SIZE; /* Round up to next whole AES adding at least one byte. */
       padding = (len2-len);
       assembly[len2 - 1] = padding;
-      traceEvent(TRACE_DEBUG, "padding = %u, seed = %016llx", padding, iv_seed);
+
+      char iv_seed_hex[2 * N2N_AES_IVEC_SIZE + 1];
+      traceEvent(TRACE_DEBUG, "padding = %u, seed = 0x%s", padding, to_hex (iv_seed, TRANSOP_AES_IV_SEED_SIZE, iv_seed_hex, 2 * N2N_AES_IVEC_SIZE + 1) );
 
       set_aes_cbc_iv(priv, enc_ivec, iv_seed);
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
       EVP_CIPHER_CTX *ctx = priv->enc_ctx;
       int evp_len;
       int evp_ciphertext_len;
@@ -229,7 +264,7 @@ static int transop_decode_aes(n2n_trans_op_t * arg,
       size_t rem=in_len;
       size_t idx=0;
       uint8_t aes_enc_ver=0;
-      uint64_t iv_seed=0;
+      uint8_t iv_seed[TRANSOP_AES_IV_SEED_SIZE];
 
       /* Get the encoding version to make sure it is supported */
       decode_uint8(&aes_enc_ver, inbuf, &rem, &idx );
@@ -238,7 +273,8 @@ static int transop_decode_aes(n2n_trans_op_t * arg,
 	/* Get the IV seed */
 	decode_buf((uint8_t *)&iv_seed, TRANSOP_AES_IV_SEED_SIZE, inbuf, &rem, &idx);
 
-	traceEvent(TRACE_DEBUG, "decode_aes %lu with seed %016llx", in_len, iv_seed);
+        char iv_seed_hex[2 * N2N_AES_IVEC_SIZE + 1];
+        traceEvent(TRACE_DEBUG, "decode_aes %lu with seed 0x%s", in_len, to_hex (iv_seed, TRANSOP_AES_IV_SEED_SIZE, iv_seed_hex, 2 * N2N_AES_IVEC_SIZE + 1) );
 
 	len = (in_len - TRANSOP_AES_PREAMBLE_SIZE);
 
@@ -248,7 +284,7 @@ static int transop_decode_aes(n2n_trans_op_t * arg,
 
 	  set_aes_cbc_iv(priv, dec_ivec, iv_seed);
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
 	  EVP_CIPHER_CTX *ctx = priv->dec_ctx;
 	  int evp_len;
 	  int evp_plaintext_len;
@@ -319,7 +355,7 @@ static int setup_aes_key(transop_aes_t *priv, const uint8_t *key, ssize_t key_si
   size_t key_mat_buf_length;
 
   /* Clear out any old possibly longer key matter. */
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
   memset(&(priv->key), 0, sizeof(priv->key) );
 #else
   memset(&(priv->enc_key), 0, sizeof(priv->enc_key) );
@@ -344,14 +380,14 @@ static int setup_aes_key(transop_aes_t *priv, const uint8_t *key, ssize_t key_si
    */
 
   if(key_size >= 65) {
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
     priv->cipher = EVP_aes_256_cbc();
 #endif
     aes_key_size_bytes = AES256_KEY_BYTES;
     SHA512(key, key_size, key_mat_buf);
     key_mat_buf_length = SHA512_DIGEST_LENGTH;
   } else if(key_size >= 44) {
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
     priv->cipher = EVP_aes_192_cbc();
 #endif
     aes_key_size_bytes = AES192_KEY_BYTES;
@@ -360,7 +396,7 @@ static int setup_aes_key(transop_aes_t *priv, const uint8_t *key, ssize_t key_si
     SHA256(key_mat_buf, SHA384_DIGEST_LENGTH, key_mat_buf + SHA384_DIGEST_LENGTH);
     key_mat_buf_length = SHA384_DIGEST_LENGTH + SHA256_DIGEST_LENGTH;
   } else {
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
     priv->cipher = EVP_aes_128_cbc();
 #endif
     aes_key_size_bytes = AES128_KEY_BYTES;
@@ -381,7 +417,7 @@ static int setup_aes_key(transop_aes_t *priv, const uint8_t *key, ssize_t key_si
   /* setup of key, used for the CBC encryption */
   aes_key_size_bits = 8 * aes_key_size_bytes;
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
   memcpy (priv->key, key_mat_buf, aes_key_size_bytes);
 #else
   AES_set_encrypt_key(key_mat_buf, aes_key_size_bits, &(priv->enc_key));
@@ -425,7 +461,7 @@ int n2n_transop_aes_cbc_init(const n2n_edge_conf_t *conf, n2n_trans_op_t *ttt) {
   }
   ttt->priv = priv;
 
-#ifdef OPENSSL_1_1
+#ifdef HAVE_OPENSSL_1_1
   /* Setup openssl's reusable evp_* contexts for encryption and decryption*/
   if(!(priv->enc_ctx = EVP_CIPHER_CTX_new())) {
     traceEvent(TRACE_ERROR, "openssl's evp_* encryption context creation: %s\n", openssl_err_as_string());