attack algorithms/event_based_detector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dpkt\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy\n",
    "import math\n",
    "# import pickle5 as pickle\n",
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import os\n",
    "import time\n",
    "from datetime import datetime\n",
    "import json\n",
    "import csv\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TransitionPeriod = 4\n",
    "amazon_ips = {}\n",
    "GAMMA = 0.25\n",
    "observation_lengths = [180, 300, 900, 1800] # in seconds\n",
    "LENGTH = 3600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_message_file_names(b, e, messages_dir):\n",
    "    all_messages_path = 'path/to/generated_messages/messages_pareto_xm_5000_alpha_0.93_max30delay_ver02'\n",
    "    all_messages_file_names = sorted(os.listdir(all_messages_path))\n",
    "    if not os.path.exists(messages_dir):\n",
    "        os.makedirs(messages_dir)\n",
    "    for i in range(len(all_messages_file_names)):\n",
    "        if i >= b and i <= e:\n",
    "            print(all_messages_file_names[i])\n",
    "            os.system('cp {} {}'.format(os.path.join(all_messages_path, all_messages_file_names[i]), os.path.join(messages_dir,all_messages_file_names[i])))\n",
    "    return\n",
    "\n",
    "\n",
    "root_dir = 'path/to/root/directory/of/traces'\n",
    "\n",
    "\n",
    "timestamps_dirs = []\n",
    "messages_dirs = []\n",
    "message_file_names = []\n",
    "packets_pickle_dirs = []\n",
    "packets = []\n",
    "dir_count = 0\n",
    "for path in sorted(Path(root_dir).iterdir()):\n",
    "    if path.is_dir() and path.name != 'aggregate' and not path.name.endswith('extra'):\n",
    "        print(path)\n",
    "        timestamps_dirs.append(os.path.join(path, 'timestamps'))\n",
    "        print('timestamps are in {}'.format(timestamps_dirs[-1]))\n",
    "        messages_dir = Path(os.path.join(path, 'messages'))\n",
    "        messages_dir.mkdir(parents=True, exist_ok=True)\n",
    "        messages_dirs.append(os.path.join(path, 'messages'))\n",
    "        print('messages are in {}'.format(messages_dirs[-1]))\n",
    "        range_begin = int(path.name.split('_')[0])\n",
    "        range_end = int(path.name.split('_')[1])\n",
    "        print(range_begin, range_end)\n",
    "        get_message_file_names(range_begin, range_end, messages_dirs[dir_count])\n",
    "        message_file_names.extend(sorted(os.listdir(messages_dirs[dir_count])))\n",
    "        print('length of message_file_names is {}'.format(len(message_file_names)))\n",
    "        packets_pickle_dirs.append(os.path.join(path, 'traces/pickles/packets'))\n",
    "        # read all packets from packets pickle and write them to 'packets'\n",
    "        pickle_name = os.listdir(packets_pickle_dirs[dir_count])[0]\n",
    "        print('loading {} ...'.format(pickle_name))\n",
    "        start = time.time()\n",
    "        with open(os.path.join(packets_pickle_dirs[dir_count], pickle_name[:-7] + '.pickle'), 'rb') as handle:\n",
    "            packets.extend(pickle.load(handle))\n",
    "        end = time.time()\n",
    "        hours, rem = divmod(end - start, 360)\n",
    "        mins, seconds = divmod(rem, 60)\n",
    "        print('loading took {:0>2}:{:0>2}:{:05.2f}'.format(int(hours), int (mins), seconds))\n",
    "        dir_count += 1\n",
    "aggregate_dir = Path(root_dir) / 'aggregate'\n",
    "aggregate_dir.mkdir(parents=True, exist_ok=True)\n",
    "results_dir = str(aggregate_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sizes_sender(messages_file, max_count=None):\n",
    "    sizes = []\n",
    "    with open(messages_file) as f:\n",
    "        data = json.load(f)\n",
    "        count = 0\n",
    "        for message in data['messages']:\n",
    "            if max_count is None or count < max_count:\n",
    "                sizes.append(int(message['size'])) # sizes are in bytes\n",
    "            else:\n",
    "                break\n",
    "            count += 1\n",
    "    return sizes\n",
    "\n",
    "def get_timestamps_sender(timestamps_file, max_count=None):\n",
    "    timestamps = []\n",
    "    with open(timestamps_file) as f:\n",
    "        lines = f.readlines()\n",
    "        count = 0\n",
    "        for line in lines:\n",
    "            if max_count is None or count < max_count:\n",
    "                line = line.split()\n",
    "                timestamps.append((float)(line[2]))\n",
    "            else:\n",
    "                break\n",
    "            count += 1\n",
    "    return timestamps\n",
    "\n",
    "def get_sizes_of_all_senders(messages_dir):\n",
    "    message_file_names = sorted(os.listdir(messages_dir))\n",
    "    all_message_sizes_of_senders = []\n",
    "    for mf_name in message_file_names:\n",
    "        message_sizes = get_sizes_sender(os.path.join(messages_dir, mf_name))\n",
    "        all_message_sizes_of_senders.append(message_sizes)\n",
    "    return all_message_sizes_of_senders\n",
    "\n",
    "def get_timestamps_of_all_senders(messages_dir, timestamps_root_dir):\n",
    "    message_file_names = sorted(os.listdir(messages_dir))\n",
    "    all_timestamps = []\n",
    "    for mf_name in message_file_names:\n",
    "        timestamps = get_timestamps_sender(os.path.join(timestamps_root_dir, 'timestamps_' + mf_name[:-5] + '.txt'))\n",
    "        all_timestamps.append(timestamps)\n",
    "    return all_timestamps\n",
    "\n",
    "def get_types_of_all_senders(messages_dir, max_count=None):\n",
    "    message_file_names = sorted(os.listdir(messages_dir))\n",
    "    all_types = []\n",
    "    for messages_file in message_file_names:\n",
    "        with open(os.path.join(messages_dir, messages_file)) as f:\n",
    "            types = []\n",
    "            data = json.load(f)\n",
    "            count = 0\n",
    "            for message in data['messages']:\n",
    "                if max_count is None or count < max_count:\n",
    "                    types.append((message['type']))\n",
    "                else:\n",
    "                    break\n",
    "                count += 1\n",
    "            all_types.append(types)\n",
    "    return all_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timestamps_of_all_senders = []\n",
    "message_sizes_of_all_senders = []\n",
    "message_types_of_all_senders = []\n",
    "for i in range(dir_count):\n",
    "    timestamps_of_all_senders.extend(get_timestamps_of_all_senders(messages_dirs[i], timestamps_dirs[i]))\n",
    "    message_sizes_of_all_senders.extend(get_sizes_of_all_senders(messages_dirs[i]))\n",
    "    message_types_of_all_senders.extend(get_types_of_all_senders(messages_dirs[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'to_amazon': False,\n",
       " 'src': '192.168.122.91',\n",
       " 'dst': '192.168.122.75',\n",
       " 'size': 64,\n",
       " 'timestamp': 1634137925.66455,\n",
       " 'protocol': 6,\n",
       " 's_port': 1080,\n",
       " 'd_port': 40940}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "packets[500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def separate_receiver_packets_non_chronological_traces(packets, timestamps_of_all_senders):\n",
    "    packets_of_all_receivers = []\n",
    "    all_beginnings = []\n",
    "    all_endings = []\n",
    "    for sender_timestamps in timestamps_of_all_senders:\n",
    "        beginning = sender_timestamps[0] - 5 #seconds\n",
    "        ending = sender_timestamps[len(sender_timestamps)-1] + 30 #seconds\n",
    "        all_beginnings.append(beginning)\n",
    "        all_endings.append(ending)\n",
    "    print(all_beginnings)\n",
    "    print(all_endings)\n",
    "    current_sender = 0\n",
    "    num_of_senders = len(timestamps_of_all_senders)\n",
    "    current_sender_received_packets = []\n",
    "    flag = 0\n",
    "    for p in packets:\n",
    "        if p.get('timestamp') > all_beginnings[current_sender] and p.get('timestamp') < all_endings[current_sender]:\n",
    "            current_sender_received_packets.append(p)\n",
    "\n",
    "        elif current_sender == num_of_senders - 1:\n",
    "            packets_of_all_receivers.append(current_sender_received_packets)\n",
    "            break\n",
    "        elif p.get('timestamp') >= all_beginnings[current_sender + 1] and p.get('timestamp') < all_endings[current_sender + 1]:\n",
    "            packets_of_all_receivers.append(current_sender_received_packets)\n",
    "            current_sender += 1\n",
    "            current_sender_received_packets = []\n",
    "            current_sender_received_packets.append(p)\n",
    "    return packets_of_all_receivers\n",
    "\n",
    "def separate_receiver_packets_chronological_traces(packets, timestamps_of_all_senders):\n",
    "    packets_of_all_receivers = []\n",
    "    all_beginnings = []\n",
    "    all_endings = []\n",
    "    for sender_timestamps in timestamps_of_all_senders:\n",
    "        beginning = sender_timestamps[0] - 5 #seconds\n",
    "        ending = sender_timestamps[len(sender_timestamps)-1] + 30 #seconds\n",
    "        all_beginnings.append(beginning)\n",
    "        all_endings.append(ending)\n",
    "    print(all_beginnings)\n",
    "    print(all_endings)\n",
    "    current_sender = 0\n",
    "    num_of_senders = len(timestamps_of_all_senders)\n",
    "    current_sender_received_packets = []\n",
    "    for p in packets:            \n",
    "        if p.get('timestamp') > all_beginnings[current_sender] and p.get('timestamp') < all_endings[current_sender]:\n",
    "            current_sender_received_packets.append(p)\n",
    "        elif p.get('timestamp') >= all_endings[current_sender]:\n",
    "            packets_of_all_receivers.append(current_sender_received_packets)\n",
    "            if current_sender < num_of_senders - 1:\n",
    "                current_sender += 1\n",
    "                current_sender_received_packets = []\n",
    "                if p.get('timestamp') > all_beginnings[current_sender] and p.get('timestamp') < all_endings[current_sender]:\n",
    "                    current_sender_received_packets.append(p)\n",
    "            else:\n",
    "                break\n",
    "    return packets_of_all_receivers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "packets_of_all_receivers = separate_receiver_packets_non_chronological_traces(packets, timestamps_of_all_senders)\n",
    "packets_towards_receivers = [[p for p in packets if (p.get('to_amazon') is False)] for packets in packets_of_all_receivers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_time(timestamps, start_time):\n",
    "    converted_timestamps = []\n",
    "    for t in timestamps:\n",
    "        converted = t - start_time\n",
    "        if converted < 0:\n",
    "            print('Error: timestamp is larger than start time with t = {} and start_time = {}'.format(t, start_time))\n",
    "        converted_timestamps.append(converted)\n",
    "    return converted_timestamps    \n",
    "\n",
    "def get_on_periods_of_sender_2(timestamps, message_sizes):\n",
    "    global TransitionPeriod\n",
    "    bursts = []\n",
    "    for t, s in zip(timestamps, message_sizes):\n",
    "        bursts.append((t, s))\n",
    "    return bursts\n",
    "\n",
    "def get_on_periods_of_user(timestamps, user_packet_sizes):\n",
    "    global TransitionPeriod\n",
    "    is_in_burst = False\n",
    "    bursts = []\n",
    "    burst_size = 0\n",
    "    last_burst_index = 0\n",
    "    for time_iter in range(0, len(timestamps)):\n",
    "        if time_iter == len(timestamps) - 1 and is_in_burst == True:\n",
    "            if burst_size > 1514:\n",
    "                bursts.append((timestamps[last_burst_index], burst_size))\n",
    "                break\n",
    "        if timestamps[time_iter] - timestamps[last_burst_index] > TransitionPeriod:\n",
    "            if is_in_burst == True:\n",
    "                if burst_size > 1514:\n",
    "                    bursts.append((timestamps[last_burst_index], burst_size))\n",
    "                is_in_burst = False\n",
    "                burst_size = 0\n",
    "        if user_packet_sizes[time_iter] > 1400:\n",
    "            if is_in_burst == False:\n",
    "                is_in_burst = True\n",
    "                last_burst_index = time_iter\n",
    "            burst_size += user_packet_sizes[time_iter]\n",
    "    return bursts\n",
    "\n",
    "def get_bursts_user(timestamps, user_packet_sizes):\n",
    "    global TransitionPeriod\n",
    "    is_in_burst = False\n",
    "    bursts = []\n",
    "    burst_size = 0\n",
    "    last_burst_index = 0\n",
    "    for time_iter in range(0, len(timestamps)):\n",
    "        if time_iter == len(timestamps) - 1 and is_in_burst == True:\n",
    "            if burst_size > 1500:\n",
    "                bursts.append((timestamps[last_burst_index], burst_size))\n",
    "                break\n",
    "        if timestamps[time_iter] - timestamps[last_burst_index] > 1:\n",
    "            if is_in_burst == True:\n",
    "                if burst_size > 1500:\n",
    "                    bursts.append((timestamps[time_iter], burst_size))\n",
    "                is_in_burst = False\n",
    "                burst_size = 0\n",
    "        if user_packet_sizes[time_iter] > 40:\n",
    "            if is_in_burst == False:\n",
    "                is_in_burst = True\n",
    "            last_burst_index = time_iter\n",
    "            burst_size += user_packet_sizes[time_iter]\n",
    "    return bursts\n",
    "\n",
    "def get_bursts_user_2(timestamps, user_packet_sizes):\n",
    "    bursts = []\n",
    "    burst_size = 0\n",
    "    for time_iter in range(0, len(timestamps) - 1):\n",
    "        ipd = timestamps[time_iter + 1] - timestamps[time_iter]\n",
    "        if ipd < 1:\n",
    "            if user_packet_sizes[time_iter] > 40:\n",
    "                burst_size += user_packet_size[time_iter]\n",
    "        else:\n",
    "            if burst_size > 20000:\n",
    "                bursts.append((timestamps[time_iter], burst_size))\n",
    "            burst_size = 0\n",
    "    return bursts\n",
    "\n",
    "def merge_channel_period_points(ChannelPeriodPoints):\n",
    "    c = []\n",
    "    time = [x[0] for x in ChannelPeriodPoints]\n",
    "    size = [x[1] for x in ChannelPeriodPoints]\n",
    "    burst_time = []\n",
    "    burst_volume = []\n",
    "    burst_volume.append(size[0])\n",
    "    burst_time.append(time[0])\n",
    "    temp = time[0]\n",
    "    for i in range(len(time)-1):\n",
    "        if time[i+1] - temp < 2:\n",
    "            burst_volume[-1] += size[i+1]\n",
    "            temp = time[i+1]\n",
    "        else:\n",
    "            burst_time.append(time[i+1])\n",
    "            burst_volume.append(size[i+1])\n",
    "            temp = time[i+1]\n",
    "    for i in range(len(burst_time)):\n",
    "        c.append((burst_time[i], burst_volume[i]))\n",
    "    return c\n",
    "\n",
    "def get_channel_points(f):\n",
    "    ChannelPeriodPoints = []\n",
    "    first_line = True\n",
    "    for line in f:\n",
    "        line = line.split()\n",
    "        if first_line:\n",
    "            find_start_time(line[1].decode('ascii'))\n",
    "            first_line = False\n",
    "            continue\n",
    "        message_type = line[4].decode('ascii')\n",
    "        if message_type == 'text':\n",
    "            continue\n",
    "        time = convert_time(line[2].decode('ascii'))\n",
    "        if line[5].decode('ascii') == 'None':\n",
    "            size = 300000\n",
    "        else:\n",
    "            size = int(line[5].decode('ascii'))\n",
    "        ChannelPeriodPoints.append((time, size))\n",
    "    if len(ChannelPeriodPoints) == 0:\n",
    "        return []\n",
    "    c = merge_channel_period_points(ChannelPeriodPoints)\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_user_bursts = []\n",
    "all_channel_bursts = [] # the same as adversary bursts in the case of one-on-one chats\n",
    "\n",
    "for i in range(0, len(packets_towards_receivers)):\n",
    "\n",
    "    print('Hour {} {}'.format(i, message_file_names[i]))\n",
    "\n",
    "    channel_timestamps = timestamps_of_all_senders[i]\n",
    "    channel_message_sizes = message_sizes_of_all_senders[i]\n",
    "\n",
    "\n",
    "    user_packet_timestamps = [p.get('timestamp') for p in packets_towards_receivers[i]]\n",
    "    user_packet_sizes = [p.get('size') for p in packets_towards_receivers[i]]\n",
    "\n",
    "    if len(user_packet_timestamps) == 0:\n",
    "        all_user_bursts.append([])\n",
    "        all_channel_bursts.append([])\n",
    "        print('no receiver timestamps')\n",
    "        continue\n",
    "    start_time = min(channel_timestamps[0], user_packet_timestamps[0])\n",
    "    print(channel_timestamps[0])\n",
    "    print(user_packet_timestamps[0])\n",
    "    print('start time: {}'.format(start_time))\n",
    "\n",
    "    converted_sender_timestamps = convert_time(channel_timestamps, start_time)\n",
    "    sender_period_points = get_on_periods_of_sender_2(converted_sender_timestamps, channel_message_sizes)\n",
    "\n",
    "    converted_user_timestamps = convert_time(user_packet_timestamps, start_time)\n",
    "    user_period_points = get_bursts_user(converted_user_timestamps, user_packet_sizes)\n",
    "\n",
    "    all_user_bursts.append(user_period_points)\n",
    "    all_channel_bursts.append(sender_period_points)\n",
    "    print ('Hour {} is done'.format(i))\n",
    "    print (\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(results_dir, 'all_user_bursts.pickle'), 'wb') as handle:\n",
    "    pickle.dump(all_user_bursts, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "with open(os.path.join(results_dir, 'all_channel_bursts.pickle'), 'wb') as handle:\n",
    "    pickle.dump(all_channel_bursts, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(results_dir, 'all_user_bursts.pickle'), 'rb') as handle:\n",
    "    all_user_bursts = pickle.load(handle)\n",
    "with open(os.path.join(results_dir, 'all_channel_bursts.pickle'), 'rb') as handle:\n",
    "    all_channel_bursts = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement the technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "DELTA = 15 # seconds\n",
    "size_threshold = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_intervals(period_points, minute):\n",
    "    global LENGTH\n",
    "    num_of_intervals = int(LENGTH / minute)\n",
    "    intervals = []\n",
    "    for i in range(num_of_intervals):\n",
    "        intervals.append([])\n",
    "        startpoint = i * minute\n",
    "        endpoint = (i + 1) * minute\n",
    "        for j in range(len(period_points)):\n",
    "            if startpoint <= period_points[j][0] and period_points[j][0] < endpoint:\n",
    "                intervals[-1].append(period_points[j])\n",
    "    return intervals\n",
    "\n",
    "def find_intervals_channel(period_points, minute, message_types):\n",
    "    global LENGTH\n",
    "    num_of_intervals = int(LENGTH / minute)\n",
    "    intervals = []\n",
    "    for i in range(num_of_intervals):\n",
    "        intervals.append([])\n",
    "        startpoint = i * minute\n",
    "        endpoint = (i + 1) * minute\n",
    "        for j in range(len(period_points)):\n",
    "            if startpoint <= period_points[j][0] and period_points[j][0] < endpoint and message_types[j] != 'text':\n",
    "                intervals[-1].append(period_points[j])\n",
    "    return intervals\n",
    "\n",
    "def find_matches(channel_interval, user_interval):\n",
    "    number_of_matches = 0\n",
    "    number_of_nonmatches = 0\n",
    "    matched_user_intervals = set()\n",
    "    for j in range(min(len(channel_interval), len(user_interval))):\n",
    "        time = channel_interval[j][0]\n",
    "        size = channel_interval[j][1]\n",
    "        is_matched = False\n",
    "        for d in range(1, DELTA):\n",
    "            for i in range(len(user_interval)):\n",
    "                if user_interval[i] in matched_user_intervals:\n",
    "                    continue\n",
    "                if abs(user_interval[i][0] - time) < d:\n",
    "                    if abs(size - user_interval[i][1]) < GAMMA * size:\n",
    "                        matched_user_intervals.add(user_interval[i])\n",
    "                        number_of_matches += 1\n",
    "                        is_matched = True\n",
    "                        break\n",
    "            else:\n",
    "                continue\n",
    "            break\n",
    "        if is_matched is False:\n",
    "            number_of_nonmatches += 1\n",
    "    return number_of_matches / float(number_of_matches + number_of_nonmatches)\n",
    "\n",
    "def detect_correlations_from_bursts(channel_bursts, user_bursts, message_types, j):\n",
    "    global observation_lengths\n",
    "    correlations = []\n",
    "    user_intervals = find_intervals(user_bursts, observation_lengths[j])\n",
    "    channel_intervals = find_intervals_channel(channel_bursts, observation_lengths[j], message_types)\n",
    "    correlation_of_intervals = detect_correlations_from_intervals(channel_intervals, user_intervals)\n",
    "    return correlation_of_intervals\n",
    "\n",
    "def detect_correlations_from_intervals(channel_intervals, user_intervals):\n",
    "    correlation = []\n",
    "    for i in range(len(channel_intervals)):\n",
    "        if len(channel_intervals[i]) == 0 and len(user_intervals[i]) == 0: # no events in either one\n",
    "            correlation.append(-1)\n",
    "        elif len(channel_intervals[i]) == 0 or len(user_intervals[i]) == 0: # only one has an event\n",
    "            correlation.append(0)\n",
    "        else:\n",
    "            correlation.append(find_matches(channel_intervals[i], user_intervals[i]))\n",
    "    return correlation\n",
    "\n",
    "def remove_empty_correlations(correlations):\n",
    "    organized = []\n",
    "    for i in range(len(correlations)):\n",
    "        organized.append([c for c in correlations[i] if c != -1])\n",
    "    return organized"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### calculate correlations for positive samples (correlated pairs of flows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_correlations_tp = []\n",
    "for j in range(len(observation_lengths)):\n",
    "    print('For {}-second interval:'.format(observation_lengths[j]))\n",
    "    corrs_tp = []\n",
    "    for i in range(len(all_user_bursts)): # for every hour\n",
    "        corrs = detect_correlations_from_bursts(all_channel_bursts[i], all_user_bursts[i], message_types_of_all_senders[i], j)\n",
    "        if corrs != -1:\n",
    "            corrs_tp.extend(corrs)\n",
    "        print (\"Hour {} is done\".format(i))\n",
    "    all_correlations_tp.append([c for c in corrs_tp if c != -1])\n",
    "with open(os.path.join(results_dir, 'corrs_tp_event_based_delta_{}.pickle'.format(DELTA)), 'wb') as handle:\n",
    "    pickle.dump(all_correlations_tp, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate correlations for negative samples (uncorrelated pairs of flows):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# calculate correlations for false samples (uncorrelated pairs of flows)\n",
    "\n",
    "all_corrs_fp = []\n",
    "for m in range(len(observation_lengths)):\n",
    "    print('For {}-second interval:'.format(observation_lengths[m]))\n",
    "    \n",
    "    corrs_fp = []\n",
    "#     for j in range(len(all_user_bursts[i])):\n",
    "    for i in range(len(all_user_bursts)):\n",
    "        for j in range(len(all_channel_bursts)):\n",
    "            if i == j:\n",
    "                continue\n",
    "            corrs = detect_correlations_from_bursts(all_channel_bursts[j], all_user_bursts[i], message_types_of_all_senders[j], m)\n",
    "            if len(corrs) == 0:\n",
    "                print('warning')\n",
    "    #             continue\n",
    "#             organized_correlations = remove_empty_correlations(corrs)\n",
    "    #         for k in range(len(organized_correlations)):\n",
    "    #             corrs_fp[k].extend(organized_correlations[k])\n",
    "            if corrs != -1:\n",
    "                corrs_fp.extend(corrs)\n",
    "            print (\"user {} with channel {} is done\".format(i, j))\n",
    "    all_corrs_fp.append([c for c in corrs_fp if c != -1])\n",
    "with open(os.path.join(results_dir, 'corrs_fp_event_based_delta_{}.pickle'.format(DELTA)), 'wb') as handle:\n",
    "    pickle.dump(all_corrs_fp, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:messaging] *",
   "language": "python",
   "name": "conda-env-messaging-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}