Revert "add LICENSE"

This reverts commit 929b40983c

.gitignore

@@ -2,4 +2,9 @@
 venv
 __pycache__
 *.log
+.idea/*
+*/.idea
+*.idea
+/.idea
 .idea/
+2024-bsc-sebastian-lenzlinger.iml

code/iottb/__main__.py

@@ -1,8 +1,13 @@
 #!/usr/bin/env python3
 import argparse
+from os import environ
+from pathlib import Path
 
-from iottb.subcommands.capture import setup_capture_parser
+from iottb.logger import logger
 from iottb.subcommands.add_device import setup_init_device_root_parser
+from iottb.subcommands.capture import setup_capture_parser
+from iottb.utils.tcpdump_utils import list_interfaces
+from definitions import IOTTB_HOME_ABS, ReturnCodes
 
 
 ######################
@@ -13,13 +18,52 @@ def setup_argparse():
     root_parser = argparse.ArgumentParser(prog='iottb')
     subparsers = root_parser.add_subparsers(title='subcommands', required=True, dest='command')
 
+    # shared options
+    root_parser.add_argument('--verbose', '-v', action='count', default=0)
+    # configure subcommands
     setup_capture_parser(subparsers)
     setup_init_device_root_parser(subparsers)
 
+    # Utility to list interfaces directly with iottb instead of relying on external tooling
+
+    interfaces_parser = subparsers.add_parser('list-interfaces', aliases=['li', 'if'],
+                                              help='List available network interfaces.')
+    interfaces_parser.set_defaults(func=list_interfaces)
+
     return root_parser
 
 
+def check_iottb_env():
+    # This makes the option '--root-dir' obsolescent # TODO How to streamline this?\
+    try:
+        iottb_home = environ['IOTTB_HOME']  # TODO WARN implicit declaration of env var name!
+    except KeyError:
+        logger.error(f"Environment variable 'IOTTB_HOME' is not set."
+                     f"Setting environment variable 'IOTTB_HOME' to '~/{IOTTB_HOME_ABS}'")
+        environ['IOTTB_HOME'] = IOTTB_HOME_ABS
+    finally:
+        if not Path(IOTTB_HOME_ABS).exists():
+            print(f'"{IOTTB_HOME_ABS}" does not exist.')
+            response = input('Do you want to create it now? [y/N]')
+            logger.debug(f'response: {response}')
+            if response.lower() != 'y':
+                logger.debug(f'Not creating "{environ['IOTTB_HOME']}"')
+                print('TODO')
+                print("Aborting execution...")
+                return ReturnCodes.ABORTED
+            else:
+                print(f'Creating "{environ['IOTTB_HOME']}"')
+                Path(IOTTB_HOME_ABS).mkdir(parents=True,
+                                           exist_ok=False)  # Should always work since in 'not exist' code path
+                return ReturnCodes.OK
+        logger.info(f'"{IOTTB_HOME_ABS}" exists.')
+        # TODO: Check that it is a valid iottb dir or can we say it is valid by definition if?
+        return ReturnCodes.OK
+
+
 def main():
+    if check_iottb_env() != ReturnCodes.OK:
+        exit(ReturnCodes.ABORTED)
     parser = setup_argparse()
     args = parser.parse_args()
     print(args)

code/iottb/definitions.py

@@ -1,6 +1,17 @@
 from datetime import datetime
 from enum import Flag, unique, global_enum
+from pathlib import Path
 
+
+''' 
+Defining IOTTB_HOME_ABS here implies that it be immutable.
+It is used here so that one could configure it. 
+But after its used in __man__ this cannot be relied upon.
+'''
+IOTTB_HOME_ABS = Path().home() / 'IOTTB.db'
+
+# TODO maybe wrap this into class to make it easier to pass along to different objects
+# But will need more refactoring
 DEVICE_METADATA_FILE = 'device_metadata.json'
 CAPTURE_METADATA_FILE = 'capture_metadata.json'
 TODAY_DATE_STRING = datetime.now().strftime('%d%b%Y').lower()  # TODO convert to function in utils or so
@@ -24,3 +35,7 @@ class ReturnCodes(Flag):
     FILE_ALREADY_EXISTS = 5
     INVALID_ARGUMENT = 6
     INVALID_ARGUMENT_VALUE = 7
+
+
+def iottb_home_abs():
+    return None

code/iottb/subcommands/add_device.py

@@ -2,15 +2,16 @@ import logging
 import pathlib
 
 from iottb import definitions
-from iottb.definitions import DEVICE_METADATA_FILE
+from iottb.definitions import DEVICE_METADATA_FILE, ReturnCodes
 from iottb.logger import logger
 from iottb.models.device_metadata_model import DeviceMetadata
-from archive.device_metadata_utils import *
 
 logger.setLevel(logging.INFO)  # Since module currently passes all tests
 
+
 def setup_init_device_root_parser(subparsers):
-    parser = subparsers.add_parser('add-device', aliases=['add-device-root', 'add'])
+    parser = subparsers.add_parser('add-device', aliases=['add-device-root', 'add'],
+                                   help='Initialize a folder for a device.')
     parser.add_argument('--root_dir', type=pathlib.Path, default=pathlib.Path.cwd())
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--guided', action='store_true', help='Guided setup', default=False)
@@ -21,7 +22,8 @@ def setup_init_device_root_parser(subparsers):
 def handle_add(args):
     logger.info(f'Add device handler called with args {args}')
 
-    args.root_dir.mkdir(parents=True, exist_ok=True)  # else metadata.save_to_file will fail TODO: unclear what to assume
+    args.root_dir.mkdir(parents=True,
+                        exist_ok=True)  # else metadata.save_to_file will fail TODO: unclear what to assume
 
     if args.guided:
         logger.debug('begin guided setup')
@@ -72,4 +74,3 @@ def guided_setup(device_root) -> DeviceMetadata:
         logger.debug(f'Device name is {device_name}')
 
     return DeviceMetadata(device_name, device_root)
-

code/iottb/subcommands/capture.py

@@ -5,7 +5,7 @@ from iottb.definitions import *
 from iottb.models.capture_metadata_model import CaptureMetadata
 from iottb.models.device_metadata_model import DeviceMetadata, dir_contains_device_metadata
 from iottb.utils.capture_utils import get_capture_src_folder, make_capture_src_folder
-
+from iottb.utils.tcpdump_utils import check_installed
 
 def setup_capture_parser(subparsers):
     parser = subparsers.add_parser('sniff', help='Sniff packets with tcpdump')
@@ -74,6 +74,7 @@ def run_tcpdump(cmd):
         p = subprocess.run(cmd, capture_output=True, text=True, check=True)
         if p.returncode != 0:
             print(f'Error running tcpdump {p.stderr}')
+            # TODO add logging
         else:
             print(f'tcpdump run successfully\n: {p.stdout}')
     except KeyboardInterrupt:
@@ -81,6 +82,9 @@ def run_tcpdump(cmd):
 
 
 def handle_capture(args):
+    if not check_installed():
+        print('Please install tcpdump first')
+        exit(ReturnCodes.ABORTED)
     assert args.device_root is not None, f'Device root directory is required'
     assert dir_contains_device_metadata(args.device_root), f'Device metadata file \'{args.device_root}\' does not exist'
     # get device metadata

code/tests/subcommands/test_add_device.py

@@ -39,6 +39,9 @@ class TestDeviceMetadataFileCreation(unittest.TestCase):
         expected_file = self.test_dir / DEVICE_METADATA_FILE
         self.assertTrue(expected_file.exists()), f'Expected file not created: {expected_file}'
 
+    def test_add_when_file_exists(self):
+        # TODO
+        pass
 
 if __name__ == '__main__':
     unittest.main()

notes/journal/2024-05-08-sun.md

@@ -0,0 +1,177 @@
+# Commands to remember + sample output
+Used commands: [[nmcli]], [[iw]], [[grep]], [[sed]]
+Resources: [Capturing Wireless LAN Packets in Monitor Mode with iw](https://sandilands.info/sgordon/capturing-wifi-in-monitor-mode-with-iw)
+Foreign BSSIDs have been made anonymous by replacing with `XX:XX:XX:XX:XX:XX`.
+## [[nmcli]]
+Useful for getting channel needed to setup monitor mode properly.
+### `nmcli dev wifi`
+```
+IN-USE  BSSID              SSID                        MODE   CHAN  RATE        SIGNAL  BARS  SECURITY  
+        XX:XX:XX:XX:XX:XX  FRITZ!Box 5490 PB           Infra  6     195 Mbit/s  75      ▂▄▆_  WPA2      
+*       4C:1B:86:D1:06:7B  LenbrO                      Infra  100   540 Mbit/s  67      ▂▄▆_  WPA2      
+        4C:1B:86:D1:06:7C  LenbrO                      Infra  6     260 Mbit/s  64      ▂▄▆_  WPA2      
+        B8:BE:F4:4D:48:17  LenbrO                      Infra  1     130 Mbit/s  62      ▂▄▆_  WPA  
+        XX:XX:XX:XX:XX:XX  --                          Infra  6     260 Mbit/s  60      ▂▄▆_  WPA2
+        XX:XX:XX:XX:XX:XX  FRITZ!Box 5490 PB           Infra  60    405 Mbit/s  37      ▂▄__  WPA2
+        XX:XX:XX:XX:XX:XX  FRITZ!Box Fon WLAN 7360 BP  Infra  1     130 Mbit/s  34      ▂▄__  WPA1 WPA2
+        XX:XX:XX:XX:XX:XX  FRITZ!Box 5490 PB           Infra  6     195 Mbit/s  34      ▂▄__  WPA2
+        XX:XX:XX:XX:XX:XX  Sunrise_Wi-Fi_09FB29        Infra  7     540 Mbit/s  34      ▂▄__  WPA2 WPA3
+        XX:XX:XX:XX:XX:XX  Madchenband                 Infra  11    260 Mbit/s  34      ▂▄__  WPA2
+        XX:XX:XX:XX:XX:XX  LenbrO                      Infra  36    270 Mbit/s  34      ▂▄__  WPA2
+        XX:XX:XX:XX:XX:XX  FibreBox_X6-01EF47          Infra  1     260 Mbit/s  32      ▂▄__  WPA2      
+        XX:XX:XX:XX:XX:XX  --                          Infra  11    260 Mbit/s  32      ▂▄__  WPA2
+        XX:XX:XX:XX:XX:XX  EEG-04666                   Infra  1     405 Mbit/s  30      ▂___  WPA2
+        XX:XX:XX:XX:XX:XX  Salt_2GHz_8A9170            Infra  11    260 Mbit/s  29      ▂___  WPA2
+        XX:XX:XX:XX:XX:XX  --                          Infra  11    260 Mbit/s  24      ▂___  WPA2
+        XX:XX:XX:XX:XX:XX  FRITZ!Box 5490 PB           Infra  60    405 Mbit/s  19      ▂___  WPA2
+```
+### `nmcli -t dev wifi`
+```
+ XX\:XX\:XX\:XX\:XX\:XX:FRITZ!Box 5490 PB:Infra:6:195 Mbit/s:79:▂▄▆_:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX::Infra:6:260 Mbit/s:75:▂▄▆_:WPA2
+ :4C\:1B\:86\:D1\:06\:7C:LenbrO:Infra:6:260 Mbit/s:74:▂▄▆_:WPA2
+*:4C\:1B\:86\:D1\:06\:7B:LenbrO:Infra:100:540 Mbit/s:72:▂▄▆_:WPA2
+ :B8\:BE\:F4\:4D\:48\:17:LenbrO:Infra:1:130 Mbit/s:65:▂▄▆_:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:Sunrise_Wi-Fi_09FB29:Infra:7:540 Mbit/s:52:▂▄__:WPA2 WPA3
+ :XX\:XX\:XX\:XX\:XX\:XX:FRITZ!Box 5490 PB:Infra:60:405 Mbit/s:50:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:FRITZ!Box Fon WLAN 7360 BP:Infra:1:130 Mbit/s:47:▂▄__:WPA1 WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:FRITZ!Box 5490 PB:Infra:6:195 Mbit/s:45:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:Zentrum der Macht:Infra:1:195 Mbit/s:44:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:FibreBox_X6-01EF47:Infra:1:260 Mbit/s:42:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:Madchenband:Infra:11:260 Mbit/s:40:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:LenbrO:Infra:36:270 Mbit/s:37:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX::Infra:11:260 Mbit/s:34:▂▄__:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:EEG-04666:Infra:1:405 Mbit/s:30:▂___:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:Salt_2GHz_8A9170:Infra:11:260 Mbit/s:29:▂___:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:FRITZ!Box 5490 PB:Infra:60:405 Mbit/s:27:▂___:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:Madchenband2.0:Infra:100:540 Mbit/s:25:▂___:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX::Infra:11:260 Mbit/s:24:▂___:WPA2
+ :XX\:XX\:XX\:XX\:XX\:XX:FibreBox_X6-01EF47:Infra:44:540 Mbit/s:20:▂___:WPA2
+```
+## [[iw]]
+### `iw dev`
+Useful to list interfaces and see which hardware they correspond to. 
+Can use that to create a monitor interface with an easier to remember name.
+```
+phy#1
+	Unnamed/non-netdev interface
+		wdev 0x100000002
+		addr 3c:21:9c:f2:e4:00
+		type P2P-device
+	Interface wlp44s0
+		ifindex 5
+		wdev 0x100000001
+		addr e6:bf:0c:3c:47:ba
+		ssid LenbrO
+		type managed
+		channel 100 (5500 MHz), width: 80 MHz, center1: 5530 MHz
+		txpower 22.00 dBm
+		multicast TXQ:
+			qsz-byt	qsz-pkt	flows	drops	marks	overlmt	hashcol	tx-bytes	tx-packets
+			0	0	0	0	0	0	0	0		0
+phy#0
+	Interface mon0
+		ifindex 7
+		wdev 0x2
+		addr a8:42:a1:8b:f4:e3
+		type monitor
+		channel 6 (2437 MHz), width: 20 MHz (no HT), center1: 2437 MHz
+		txpower 20.00 dBm
+	Interface wlp0s20f0u6
+		ifindex 4
+		wdev 0x1
+		addr a8:42:a1:8b:f4:e3
+		type monitor
+		channel 6 (2437 MHz), width: 20 MHz (no HT), center1: 2437 MHz
+		txpower 20.00 dBm
+		multicast TXQ:
+			qsz-byt	qsz-pkt	flows	drops	marks	overlmt	hashcol	tx-bytes	tx-packets
+			0	0	0	0	0	0	0	0		0
+
+```
+Here, `phy#1` is my laptops built-in WiFi card, and `phy#0` is a WiFi USB adapter. 
+### `iw [phy phy<index> | phy#<index>] info | grep -f monitor -B 10`
+```
+➜ iw phy phy0 info | fgrep monitor -B 10
+		* CMAC-256 (00-0f-ac:13)
+		* GMAC-128 (00-0f-ac:11)
+		* GMAC-256 (00-0f-ac:12)
+	Available Antennas: TX 0x3 RX 0x3
+	Configured Antennas: TX 0x3 RX 0x3
+	Supported interface modes:
+		 * IBSS
+		 * managed
+		 * AP
+		 * AP/VLAN
+		 * monitor
+--
+		 * register_beacons
+		 * start_p2p_device
+		 * set_mcast_rate
+		 * connect
+		 * disconnect
+		 * set_qos_map
+		 * set_multicast_to_unicast
+		 * set_sar_specs
+	software interface modes (can always be added):
+		 * AP/VLAN
+		 * monitor
+```
+Can do better
+### `iw phy#0 info | grep monitor`
+```
+		 * monitor
+		 * monitor
+```
+Concise but possible need more context to be sure?
+### `iw phy phy0 info | sed -n '/software interface modes/,/monitor/p'`
+More concise but with good context. Assuming only sw interfaces need to support monitor mode
+```
+	software interface modes (can always be added):
+		 * AP/VLAN
+		 * monitor
+```
+### Getting a monitor interface
+```
+iw phy#0 interface add mon0 type monitor
+```
+Add a easy interface to wifi hw and make it a monitor. Can check again with 'iw dev' to make sure it is really in monitor mode. If there is an other interface it must be taken down or deleted e.g with 
+```
+iw dev <phy#0 other interface> del # or
+ip link set <phy#0 other interface> down
+```
+Then to enable `mon0` interface,
+```
+ip link set mon0 up
+```
+To effectively capture packets, we should set the interface to the correct frequency. For this we get the channel e.g. via the above mentioned `nmcli dev wifi`. We can see that, e.g. the BSSID I am connected to (marked with `*`) is on channel 100. We can also see that it there is also a BSSID belonging to the same SSID with the interface on channel 6. I.e., it is running one interface in  2.4 GHz (802.11b/g/n/ax/be) and one in 5 GHz (802.11a/h/n/ac/ax/be). We chose which which channel to tune our `mon0` interface to, then we can lookup what the center frequency is on [wikipedia(List of Wifi Channels)](https://en.wikipedia.org/wiki/List_of_WLAN_channels). E.g. for channel 6 (i.e. 2.4 GHz radio) we see that the center frequency is 2437. We set our interface to that frequency:
+```
+iw dev mon0 set freq 2437
+```
+Now double check that the interface is in monitor mode and tunedto the correct frequency:
+```
+iw dev mon0 info
+```
+Should give an output like
+```
+Interface mon0
+	ifindex 7
+	wdev 0x2
+	addr a8:42:a1:8b:f4:e3
+	type monitor
+	wiphy 0
+	channel 6 (2437 MHz), width: 20 MHz (no HT), center1: 2437 MHz
+	txpower 20.00 dBm
+```
+This concludes preparing the wifi card for packet capture in monitor mode.
+### [remarks]
+- `sudo` is probably required for these commands
+- These network tools are what is available on fedora 40, on $(uname -r)= 6.8.8 Linux Kernel. It might be that other OSs still use older tools, which are being phased out. But other operating systems might still be using older versions of these commands. For a table on how they match up, see  [this](https://www.tecmint.com/deprecated-linux-networking-commands-and-their-replacements/) recent article (July 2023), according to which the old commands are even deprecated in recent Debian and Ubuntu releases.
+- If smth is not working run `rfkill list` to check device is blocked. If it is, `rfkill unblock 0`, where `0` is the same index used above and represents `phy0` /`phy#0`.
+- To ensure that [[NetworkManager]] not managing you card, `nmcli device set wlp0s20f0u6 managed no` if the interface is called `wlp0s20f0u6`. Check with `nmcli dev`, the STATE should be "unmanaged".
+- See resources on how to put interface/wifi hardware back into managed mode, if you need the card for personal use.
+
+# Important
+Monitor mode is actually completely useless, unless we can observe the EAPOL handshake. That means the Wifi AP should be using WPA/WPA2 with psk. Also we need to know the SSID and passphrase. So it is still better if we can setup an environment where we can just do port mirroring from the wifi router, or setup ourselves in AP mode, but then we need to be able to bridge to the internet somehow, which I haven't managed reliably. Have done some testing on raspberry pi seemed to work. But raspberry pi sometimes goes to sleep so the AP goes down which means the IoT device loses connection.
+
+If we happen to know the MAC address we need, then in wireshark we can filter `wlan.addr == [MAC]`. In tcpdump we can use the filter 

notes/journal/2024-05-15-wed.md

@@ -0,0 +1,14 @@
+# `IOTTB_HOME`
+I introduced the environment variable `IOTTB_HOME` into the code. It is used to configure where the root of a iottb database is. #TODO this means that some code needs refactoring. But, I think it will streamline the code. The path in `IOTTB_HOME` shall be used to define the database root. Then, all the code handling adding devices and running captures can rely on the fact that a canonical home exists. Unfortunately I've hard coded quite a bit of ad-hoc configuration to use `Path.cwd()`, i.e. the current working directory, by default. So there will be some refactoring involved in switching over to using `IOTTB_HOME`s value as the default path. 
+
+# Adding Functionality
+## Quick and dirty capture
+I want to have a mode which just takes a command and runs it directly with its arguments.
+The question is weather to only allow a preconfigured list of commands or in principle allow any command to be passed and write the output. I tend toward providing a subcommand for each utility we want to support. The question is what to do about the syntax errors of those commands. Maybe the thing to do is only write a file into the db if the command runs successfully. 
+### Refactoring the tcpdump capture
+With the above idea it would be possible to also refactor or rewrite how tcpdump is called completely. But, the command has a lot of options and maybe its better also offer some guidance to users via `-h`, e.g. to only input the needed and correct filters for example. Choosing the wrong filter could make the capture potentially useless and one might only see that after the capture has completed. 
+## Converting pcap to csv
+I want an option such that one can automatically convert a captures resulting file into a csv. Probably will focus on tcpdump for now, since other tools like [[mitmproxy]] have different output files. 
+
+## Defining Experiment
+I want a pair of commands that 1. provide a guided cli interface to define an experiment and 2. to run that  experiment -> Here [Collective Knowledge Framework](https://github.com/mlcommons/ck) might actually come in handy. The already have tooling for setting up and defining aspects of experiments so that they become reproducible. So maybe one part of the `iottb` as a tool would be to write the correct json files into the directory which contain the informatin on how the command was run. Caveat: All all option values are the same, basically only, if it was used or not (flagging options) or that it was used (e.g. an ip address was used in the filter but the specific value of the ip is of no use for reproducing). Also, Collective Minds tooling relies very common ML algos/framework and static data. So maybe this only comes into play after a capture has been done. So maybe a feature extraction tool (see [[further considerations#Usage paths/ Workflows]]) should create the data and built the database separately.