Remove conflicting paths before merge'

.gitignore

@@ -1,10 +1,36 @@
-.obsidian
-venv
 __pycache__
+.venv
+iottb.egg-info
+.idea
 *.log
-.idea/*
+logs/
-*/.idea
+*.pyc
-*.idea
+.obsidian
-/.idea
+
-.idea/
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
-2024-bsc-sebastian-lenzlinger.iml
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+.private/
+*.pcap

.idea/2024-bsc-sebastian-lenzlinger.iml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="PLAIN" />
-    <option name="myDocStringFormat" value="Plain" />
-  </component>
-</module>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Black">
-    <option name="sdkName" value="Python 3.12 (pythonProject)" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (venv)" project-jdk-type="Python SDK" />
-</project>

.idea/webResources.xml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="WebResourcesPaths">
-    <contentEntries>
-      <entry url="file://$PROJECT_DIR$">
-        <entryData>
-          <resourceRoots>
-            <path value="file://$PROJECT_DIR$/data" />
-          </resourceRoots>
-        </entryData>
-      </entry>
-    </contentEntries>
-  </component>
-</project>

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Sebastian Lenzlinger
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

archive/functions_dump.py

@@ -30,3 +30,30 @@ def setup_sniff_parser(subparsers):
 def setup_pcap_filter_parser(parser_sniff):
     parser_pcap_filter = parser_sniff.add_argument_parser('pcap-filter expression')
     pass
+
+def check_iottb_env():
+    # This makes the option '--root-dir' obsolescent # TODO How to streamline this?\
+    try:
+        iottb_home = environ['IOTTB_HOME']  # TODO WARN implicit declaration of env var name!
+    except KeyError:
+        logger.error(f"Environment variable 'IOTTB_HOME' is not set."
+                     f"Setting environment variable 'IOTTB_HOME' to '~/{IOTTB_HOME_ABS}'")
+        environ['IOTTB_HOME'] = IOTTB_HOME_ABS
+    finally:
+        if not Path(IOTTB_HOME_ABS).exists():
+            print(f'"{IOTTB_HOME_ABS}" does not exist.')
+            response = input('Do you want to create it now? [y/N]')
+            logger.debug(f'response: {response}')
+            if response.lower() != 'y':
+                logger.debug(f'Not setting "IOTTB_HOME"')
+                print('TODO')
+                print("Aborting execution...")
+                return ReturnCodes.ABORTED
+            else:
+                print(f'Setting environment variable IOTTB_HOME""')
+                Path(IOTTB_HOME_ABS).mkdir(parents=True,
+                                           exist_ok=False)  # Should always work since in 'not exist' code path
+                return ReturnCodes.SUCCESS
+        logger.info(f'"{IOTTB_HOME_ABS}" exists.')
+        # TODO: Check that it is a valid iottb dir or can we say it is valid by definition if?
+        return ReturnCodes.SUCCESS

archive/iottb/__main__.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import argparse
+from os import environ
+from pathlib import Path
+import logging
+from archive.iottb.subcommands.add_device import setup_init_device_root_parser
+# from iottb.subcommands.capture import setup_capture_parser
+from iottb.subcommands.sniff import setup_sniff_parser
+from iottb.utils.tcpdump_utils import list_interfaces
+from iottb.logger import setup_logging
+
+logger = logging.getLogger('iottbLogger.__main__')
+logger.setLevel(logging.DEBUG)
+
+
+######################
+# Argparse setup
+######################
+def setup_argparse():
+    # create top level parser
+    root_parser = argparse.ArgumentParser(prog='iottb')
+    # shared options
+    root_parser.add_argument('--verbose', '-v', action='count', default=0)
+    root_parser.add_argument('--script-mode', action='store_true', help='Run in script mode (non-interactive)')
+    # Group of args w.r.t iottb.db creation
+    group = root_parser.add_argument_group('database options')
+    group.add_argument('--db-home', default=Path.home() / 'IoTtb.db')
+    group.add_argument('--config-home', default=Path.home() / '.config' / 'iottb.conf', type=Path, )
+    group.add_argument('--user', default=Path.home().stem, type=Path, )
+
+    # configure subcommands
+    subparsers = root_parser.add_subparsers(title='subcommands', required=True, dest='command')
+    # setup_capture_parser(subparsers)
+    setup_init_device_root_parser(subparsers)
+    setup_sniff_parser(subparsers)
+    # Utility to list interfaces directly with iottb instead of relying on external tooling
+
+    interfaces_parser = subparsers.add_parser('list-interfaces', aliases=['li', 'if'],
+                                              help='List available network interfaces.')
+    interfaces_parser.set_defaults(func=list_interfaces)
+
+    return root_parser
+
+
+###
+# Where put ?!
+###
+class IoTdb:
+    def __init__(self, db_home=Path.home() / 'IoTtb.db', iottb_config=Path.home() / '.conf' / 'iottb.conf',
+                 user=Path.home().stem):
+        self.db_home = db_home
+        self.config_home = iottb_config
+        self.default_filters_home = db_home / 'default_filters'
+        self.user = user
+
+    def create_db(self, mode=0o777, parents=False, exist_ok=False):
+        logger.info(f'Creating db at {self.db_home}')
+        try:
+            self.db_home.mkdir(mode=mode, parents=parents, exist_ok=exist_ok)
+        except FileExistsError:
+            logger.error(f'Database path already at {self.db_home} exists and is not a directory')
+        finally:
+            logger.debug(f'Leaving finally clause in create_db')
+
+    def create_device_tree(self, mode=0o777, parents=False, exist_ok=False):
+        logger.info(f'Creating device tree at {self.db_home / 'devices'}')
+        #TODO
+
+    def parse_db_config(self):
+        pass
+
+    def parse_iottb_config(self):
+        pass
+
+    def get_known_devices(self):
+        pass
+
+
+def iottb_db_exists(db_home=Path.home() / 'IoTtb.db'):
+    res = db_home.is_dir()
+
+
+def main():
+    logger.debug(f'Pre setup_argparse()')
+    parser = setup_argparse()
+    logger.debug('Post setup_argparse().')
+    args = parser.parse_args()
+    logger.debug(f'Args parsed: {args}')
+    if args.command:
+        try:
+            args.func(args)
+        except KeyboardInterrupt:
+            print('Received keyboard interrupt. Exiting...')
+            exit(1)
+        except Exception as e:
+            logger.debug(f'Error in main: {e}')
+            print(f'Error: {e}')
+    # create_capture_directory(args.device_name)
+
+
+if __name__ == '__main__':
+    setup_logging()
+    logger.debug("Debug level is working")
+    logger.info("Info level is working")
+    logger.warning("Warning level is working")
+
+    main()

archive/iottb/logger.py

@@ -0,0 +1,35 @@
+import logging
+import sys
+import os
+from logging.handlers import RotatingFileHandler
+
+
+def setup_logging():
+    # Ensure the logs directory exists
+    log_directory = 'logs'
+    if not os.path.exists(log_directory):
+        os.makedirs(log_directory)
+
+    # Create handlers
+    file_handler = RotatingFileHandler(os.path.join(log_directory, 'iottb.log'), maxBytes=1048576, backupCount=5)
+    console_handler = logging.StreamHandler(sys.stdout)
+
+    # Create formatters and add it to handlers
+    file_fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    console_fmt = logging.Formatter(
+        '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(funcName)s - %(message)s')
+
+    file_handler.setFormatter(file_fmt)
+    console_handler.setFormatter(console_fmt)
+
+    # Get the root logger and add handlers
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG)
+    root_logger.addHandler(file_handler)
+    root_logger.addHandler(console_handler)
+
+    # Prevent propagation to the root logger to avoid duplicate logs
+    root_logger.propagate = False
+
+
+setup_logging()

archive/iottb/models/capture_metadata_model.py

@@ -6,17 +6,20 @@ from typing import Optional
 
 from iottb.definitions import ReturnCodes, CAPTURE_METADATA_FILE
 from iottb.models.device_metadata_model import DeviceMetadata
-from iottb.logger import logger
+import logging
+
+logger = logging.getLogger('iottbLogger.capture_metadata_model')
+logger.setLevel(logging.DEBUG)
 
 
 class CaptureMetadata:
     # Required Fields
     device_metadata: DeviceMetadata
-    capture_id: str = lambda: str(uuid.uuid4())
+
     device_id: str
     capture_dir: Path
     capture_file: str
-    capture_date: str = lambda: datetime.now().strftime('%d-%m-%YT%H:%M:%S').lower()
+
 
     # Statistics
     start_time: str
@@ -39,7 +42,8 @@ class CaptureMetadata:
     def __init__(self, device_metadata: DeviceMetadata, capture_dir: Path):
         logger.info(f'Creating CaptureMetadata model from DeviceMetadata: {device_metadata}')
         self.device_metadata = device_metadata
-
+        self.capture_id = str(uuid.uuid4())
+        self.capture_date = datetime.now().strftime('%d-%m-%YT%H:%M:%S').lower()
         self.capture_dir = capture_dir
         assert capture_dir.is_dir(), f'Capture directory {capture_dir} does not exist'
 
@@ -47,7 +51,7 @@ class CaptureMetadata:
         logger.info(f'Building capture file name')
         if self.app is None:
             logger.debug(f'No app specified')
-            prefix = self.device_metadata.device_short_name
+            prefix = "iphone-14" #self.device_metadata.device_short_name
         else:
             logger.debug(f'App specified: {self.app}')
             assert str(self.app).strip() not in {'', ' '}, f'app is not a valid name: {self.app}'

archive/iottb/models/device_metadata_model.py

@@ -6,7 +6,10 @@ from typing import Optional, List
 
 # iottb modules
 from iottb.definitions import ReturnCodes, DEVICE_METADATA_FILE
-from iottb.logger import logger
+import logging
+
+logger = logging.getLogger('iottbLogger.device_metadata_model')
+logger.setLevel(logging.DEBUG)
 # 3rd party libs
 
 IMMUTABLE_FIELDS = {'device_name', 'device_short_name', 'device_id', 'date_created'}

archive/iottb/subcommands/add_device.py

@@ -1,18 +1,21 @@
 import logging
+import os
 import pathlib
 
 from iottb import definitions
 from iottb.definitions import DEVICE_METADATA_FILE, ReturnCodes
-from iottb.logger import logger
 from iottb.models.device_metadata_model import DeviceMetadata
 
-logger.setLevel(logging.INFO)  # Since module currently passes all tests
+# logger.setLevel(logging.INFO)  # Since module currently passes all tests
-
+logger = logging.getLogger('iottbLogger.add_device')
+logger.setLevel(logging.INFO)
 
 def setup_init_device_root_parser(subparsers):
+    #assert os.environ['IOTTB_HOME'] is not None, "IOTTB_HOME environment variable is not set"
     parser = subparsers.add_parser('add-device', aliases=['add-device-root', 'add'],
                                    help='Initialize a folder for a device.')
-    parser.add_argument('--root_dir', type=pathlib.Path, default=pathlib.Path.cwd())
+    parser.add_argument('--root_dir', type=pathlib.Path,
+                        default=definitions.IOTTB_HOME_ABS)  # TODO: Refactor code to not use this or handle iottb here
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--guided', action='store_true', help='Guided setup', default=False)
     group.add_argument('--name', action='store', type=str, help='name of device')
@@ -20,14 +23,12 @@ def setup_init_device_root_parser(subparsers):
 
 
 def handle_add(args):
+    # TODO: This whole function should be refactored into using the fact that IOTTB_HOME is set, and the dir exists
     logger.info(f'Add device handler called with args {args}')
 
-    args.root_dir.mkdir(parents=True,
-                        exist_ok=True)  # else metadata.save_to_file will fail TODO: unclear what to assume
-
     if args.guided:
         logger.debug('begin guided setup')
-        metadata = guided_setup(args.root_dir)
+        metadata = guided_setup(args.root_dir)  # TODO refactor to use IOTTB_HOME
         logger.debug('guided setup complete')
     else:
         logger.debug('Setup through passed args: setup')
@@ -36,7 +37,7 @@ def handle_add(args):
             return ReturnCodes.ERROR
         metadata = DeviceMetadata(args.name, args.root_dir)
 
-    file_path = args.root_dir / DEVICE_METADATA_FILE
+    file_path = args.root_dir / DEVICE_METADATA_FILE    # TODO IOTTB_HOME REFACTOR
     if file_path.exists():
         print('Directory already contains a metadata file. Aborting.')
         return ReturnCodes.ABORTED

archive/iottb/subcommands/capture.py

@@ -2,11 +2,15 @@ import subprocess
 from pathlib import Path
 
 from iottb.definitions import *
+import logging
 from iottb.models.capture_metadata_model import CaptureMetadata
 from iottb.models.device_metadata_model import DeviceMetadata, dir_contains_device_metadata
 from iottb.utils.capture_utils import get_capture_src_folder, make_capture_src_folder
 from iottb.utils.tcpdump_utils import check_installed
 
+logger = logging.getLogger('iottbLogger.capture')
+logger.setLevel(logging.DEBUG)
+
 def setup_capture_parser(subparsers):
     parser = subparsers.add_parser('sniff', help='Sniff packets with tcpdump')
     # metadata args
@@ -33,7 +37,7 @@ def setup_capture_parser(subparsers):
                                       help='Please see tcpdump manual for details. Unused by default.')
 
     cap_size_group = parser.add_mutually_exclusive_group(required=False)
-    cap_size_group.add_argument('-c', '--count', type=int, help='Number of packets to capture.', default=1000)
+    cap_size_group.add_argument('-c', '--count', type=int, help='Number of packets to capture.', default=10)
     cap_size_group.add_argument('--mins', type=int, help='Time in minutes to capture.', default=1)
 
     parser.set_defaults(func=handle_capture)
@@ -88,6 +92,7 @@ def handle_capture(args):
     assert args.device_root is not None, f'Device root directory is required'
     assert dir_contains_device_metadata(args.device_root), f'Device metadata file \'{args.device_root}\' does not exist'
     # get device metadata
+    logger.info(f'Device root directory: {args.device_root}')
     if args.safe and not dir_contains_device_metadata(args.device_root):
         print(f'Supplied folder contains no device metadata. '
               f'Please setup a device root directory before using this command')
@@ -98,6 +103,7 @@ def handle_capture(args):
     else:
         name = input('Please enter a device name: ')
         args.device_root.mkdir(parents=True, exist_ok=True)
+
         device_data = DeviceMetadata(name, args.device_root)
     # start constructing environment for capture
     capture_dir = get_capture_src_folder(args.device_root)
@@ -152,7 +158,7 @@ def build_tcpdump_args(args, cmd, capture_metadata: CaptureMetadata):
 
     capture_metadata.build_capture_file_name()
     cmd.append('-w')
-    cmd.append(capture_metadata.capture_file)
+    cmd.append(str(capture_metadata.capture_dir) + "/" + capture_metadata.capture_file)
 
     if args.safe:
         cmd.append(f'host {args.device_ip}')  # if not specified, filter 'any' implied by tcpdump
@@ -160,7 +166,6 @@ def build_tcpdump_args(args, cmd, capture_metadata: CaptureMetadata):
 
     return cmd
 
-
 # def capture_file_cmd(args, cmd, capture_dir, capture_metadata: CaptureMetadata):
 #     capture_file_prefix = capture_metadata.get_device_metadata().get_device_short_name()
 #     if args.app_name is not None:

archive/iottb/subcommands/sniff.py

@@ -0,0 +1,63 @@
+import subprocess
+import logging
+
+
+logger = logging.getLogger('iottbLogger.capture')
+logger.setLevel(logging.DEBUG)
+class Sniffer:
+    def __init__(self):
+        pass
+
+
+def setup_sniff_parser(subparsers):
+    parser = subparsers.add_parser('sniff', help='Sniff packets with tcpdump')
+    # metadata args
+    parser.add_argument('-a', '--addr', help='IP or MAC address of IoT device')
+    # tcpdump args
+    parser.add_argument('--app', help='Application name to sniff', default=None)
+
+    parser_sniff_tcpdump = parser.add_argument_group('tcpdump arguments')
+
+    parser_sniff_tcpdump.add_argument('-i', '--interface', help='Interface to capture on.', dest='capture_interface',
+                                      required=True)
+    parser_sniff_tcpdump.add_argument('-I', '--monitor-mode', help='Put interface into monitor mode',
+                                      action='store_true')
+    parser_sniff_tcpdump.add_argument('-n', help='Deactivate name resolution. True by default.',
+                                      action='store_true', dest='no_name_resolution')
+    parser_sniff_tcpdump.add_argument('-#', '--number',
+                                      help='Print packet number at beginning of line. True by default.',
+                                      action='store_true')
+    parser_sniff_tcpdump.add_argument('-e', help='Print link layer headers. True by default.',
+                                      action='store_true', dest='print_link_layer')
+    parser_sniff_tcpdump.add_argument('-t', action='count', default=0,
+                                      help='Please see tcpdump manual for details. Unused by default.')
+
+    cap_size_group = parser.add_mutually_exclusive_group(required=False)
+    cap_size_group.add_argument('-c', '--count', type=int, help='Number of packets to capture.', default=10)
+    cap_size_group.add_argument('--mins', type=int, help='Time in minutes to capture.', default=1)
+
+    parser.set_defaults(func=sniff)
+
+
+def parse_addr(addr):
+    #TODO Implement
+    pass
+
+
+def sniff(args):
+    if args.addr is None:
+        print('You must supply either a MAC or IP(v4) address to use this tool!')
+        logger.info("Exiting on account of missing MAC/IP.")
+        exit(1)
+    else:
+        (type, value) = parse_addr(args.addr)
+        #TODO Get this party started
+
+def sniff_tcpdump(args, filter):
+    pass
+
+def sniff_mitmproxy(args, filter):
+    pass
+
+def sniff_raw(cmd,args):
+    pass

archive/iottb/utils/tcpdump_utils.py

@@ -15,12 +15,12 @@ def ensure_installed():
         raise RuntimeError('tcpdump is not installed. Please install it to continue.')
 
 
-def list_interfaces() -> str:
+def list_interfaces(args) -> str:
     """List available network interfaces using tcpdump."""
     ensure_installed()
     try:
         result = subprocess.run(['tcpdump', '--list-interfaces'], capture_output=True, text=True, check=True)
-        return result.stdout
+        print(result.stdout)
     except subprocess.CalledProcessError as e:
         print(f'Failed to list interfaces: {e}')
         return ''

archive/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = 'iottb'
+version = '0.1.0'
+authors = [{name = "Sebastian Lenzlinger", email = "sebastian.lenzlinger@unibas.ch"}]
+description = "Automation Tool for Capturing Network packets of IoT devices."
+requires-python = ">=3.8"
+
+[tool.setuptools]
+packages = ["iottb"]
+
+[project.scripts]
+iottb = "iottb.__main__:main"

code/iottb-project/README.md

@@ -0,0 +1,9 @@
+# Iottb
+## Basic Invocation
+
+## Configuration
+### Env Vars
+- IOTTB_CONF_HOME
+
+By setting this variable you control where the basic iottb application
+configuration should be looked for

code/iottb-project/docs/command_reference.txt

@@ -0,0 +1,110 @@
+Usage: iottb [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  -v, --verbosity  Set verbosity  [default: 0; 0<=x<=3]
+  -d, --debug      Enable debug mode
+  --dry-run        [default: True]
+  --cfg-file PATH  Path to iottb config file  [default:
+                   /home/seb/.config/iottb/iottb.cfg]
+  --help           Show this message and exit.
+
+Commands:
+  add-device           Add a device to a database
+  init-db
+  rm-cfg               Removes the cfg file from the filesystem.
+  rm-dbs               Removes ALL(!) databases from the filesystem if...
+  set-key-in-table-to  Edit config or metadata files.
+  show-all             Show everything: configuration, databases, and...
+  show-cfg             Show the current configuration context
+  sniff                Sniff packets with tcpdump
+Usage: iottb init-db [OPTIONS]
+
+Options:
+  -d, --dest PATH                 Location to put (new) iottb database
+  -n, --name TEXT                 Name of new database.  [default: iottb.db]
+  --update-default / --no-update-default
+                                  If new db should be set as the new default
+                                  [default: update-default]
+  --help                          Show this message and exit.
+Usage: iottb add-device [OPTIONS]
+
+  Add a device to a database
+
+Options:
+  --dev, --device-name TEXT   The name of the device to be added. If this
+                              string contains spaces or other special
+                              characters               normalization is
+                              performed to derive a canonical name  [required]
+  --db, --database DIRECTORY  Database in which to add this device. If not
+                              specified use default from config.  [env var:
+                              IOTTB_DB]
+  --guided                    Add device interactively  [env var:
+                              IOTTB_GUIDED_ADD]
+  --help                      Show this message and exit.
+Usage: iottb sniff [OPTIONS] [TCPDUMP-ARGS] [DEVICE]
+
+  Sniff packets with tcpdump
+
+Options:
+  Testbed sources: 
+    --db, --database TEXT  Database of device. Only needed if not current
+                           default.  [env var: IOTTB_DB]
+    --app TEXT             Companion app being used during capture
+  Runtime behaviour: 
+    --unsafe               Disable checks for otherwise required options.
+                           [env var: IOTTB_UNSAFE]
+    --guided               [env var: IOTTB_GUIDED]
+    --pre TEXT             Script to be executed before main  command is
+                           started.
+    --post TEXT            Script to be executed upon completion of main
+                           command.
+  Tcpdump options: 
+    -i, --interface TEXT   Network interface to capture on.If not specified
+                           tcpdump tries to find and appropriate one.  [env
+                           var: IOTTB_CAPTURE_INTERFACE]
+    -a, --address TEXT     IP or MAC address to filter packets by.  [env var:
+                           IOTTB_CAPTURE_ADDRESS]
+    -I, --monitor-mode     Put interface into monitor mode.
+    --ff TEXT              tcpdump filter as string or file path.  [env var:
+                           IOTTB_CAPTURE_FILTER]
+    -#, --print-pacno      Print packet number at beginning of line. True by
+                           default.  [default: True]
+    -e, --print-ll         Print link layer headers. True by default.
+    -c, --count INTEGER    Number of packets to capture.  [default: 1000]
+  --help                   Show this message and exit.
+Utility Commands mostly for development
+Usage: iottb rm-cfg [OPTIONS]
+
+  Removes the cfg file from the filesystem.
+
+  This is mostly a utility during development. Once non-standard database
+  locations are implemented, deleting this would lead to iottb not being able
+  to find them anymore.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.
+Usage: iottb rm-dbs [OPTIONS]
+
+  Removes ALL(!) databases from the filesystem if they're empty.
+
+  Development utility currently unfit for use.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.
+Usage: iottb show-cfg [OPTIONS]
+
+  Show the current configuration context
+
+Options:
+  --cfg-file PATH  Path to the config file  [default:
+                   /home/seb/.config/iottb/iottb.cfg]
+  -pp              Pretty Print
+  --help           Show this message and exit.
+Usage: iottb show-all [OPTIONS]
+
+  Show everything: configuration, databases, and device metadata
+
+Options:
+  --help  Show this message and exit.

code/iottb-project/docs/help.txt

@@ -0,0 +1,38 @@
+Usage: iottb [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  -v, --verbosity  Set verbosity  [default: 0; 0<=x<=3]
+  -d, --debug      Enable debug mode
+  --dry-run        [default: True]
+  --cfg-file PATH  Path to iottb config file  [default:
+                   /home/seb/.config/iottb/iottb.cfg]
+  --help           Show this message and exit.
+
+Commands:
+  add-device           Add a device to a database
+  init-db
+  rm-cfg               Removes the cfg file from the filesystem.
+  rm-dbs               Removes ALL(!) databases from the filesystem if...
+  set-key-in-table-to  Edit config or metadata files.
+  show-all             Show everything: configuration, databases, and...
+  show-cfg             Show the current configuration context
+  sniff                Sniff packets with tcpdump
+Usage: iottb [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  -v, --verbosity  Set verbosity  [default: 0; 0<=x<=3]
+  -d, --debug      Enable debug mode
+  --dry-run        [default: True]
+  --cfg-file PATH  Path to iottb config file  [default:
+                   /home/seb/.config/iottb/iottb.cfg]
+  --help           Show this message and exit.
+
+Commands:
+  add-device           Add a device to a database
+  init-db
+  rm-cfg               Removes the cfg file from the filesystem.
+  rm-dbs               Removes ALL(!) databases from the filesystem if...
+  set-key-in-table-to  Edit config or metadata files.
+  show-all             Show everything: configuration, databases, and...
+  show-cfg             Show the current configuration context
+  sniff                Sniff packets with tcpdump

code/iottb-project/docs/help_messages.md

@@ -0,0 +1,142 @@
+# Main Command: `iottb`
+
+ Usage: `iottb  [OPTIONS] COMMAND [ARGS]...`
+
+Options:
+  -v, --verbosity  Set verbosity  [0<=x<=3] \n
+  -d, --debug      Enable debug mode
+  --dry-run
+  --cfg-file PATH  Path to iottb config file
+  --help           Show this message and exit.
+
+Commands:
+  add-device           Add a device to a database
+  init-db
+  rm-cfg               Removes the cfg file from the filesystem.
+  rm-dbs               Removes ALL(!) databases from the filesystem if...
+  set-key-in-table-to  Edit config or metadata files.
+  show-all             Show everything: configuration, databases, and...
+  show-cfg             Show the current configuration context
+  sniff                Sniff packets with tcpdump
+
+
+Command: init-db
+ Usage:  [OPTIONS]
+
+Options:
+  -d, --dest PATH                 Location to put (new) iottb database
+  -n, --name TEXT                 Name of new database.
+  --update-default / --no-update-default
+                                  If new db should be set as the new default
+  --help                          Show this message and exit.
+
+
+Command: rm-cfg
+ Usage:  [OPTIONS]
+
+  Removes the cfg file from the filesystem.
+
+  This is mostly a utility during development. Once non-standard database
+  locations are implemented, deleting this would lead to iottb not being able
+  to find them anymore.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.
+
+
+Command: set-key-in-table-to
+ Usage:  [OPTIONS]
+
+  Edit config or metadata files. TODO: Implement
+
+Options:
+  --file TEXT
+  --table TEXT
+  --key TEXT
+  --value TEXT
+  --help        Show this message and exit.
+
+
+Command: rm-dbs
+ Usage:  [OPTIONS]
+
+  Removes ALL(!) databases from the filesystem if they're empty.
+
+  Development utility currently unfit for use.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.
+
+
+Command: add-device
+ Usage:  [OPTIONS]
+
+  Add a device to a database
+
+Options:
+  --dev, --device-name TEXT   The name of the device to be added. If this
+                              string contains spaces or other special
+                              characters               normalization is
+                              performed to derive a canonical name  [required]
+  --db, --database DIRECTORY  Database in which to add this device. If not
+                              specified use default from config.  [env var:
+                              IOTTB_DB]
+  --guided                    Add device interactively  [env var:
+                              IOTTB_GUIDED_ADD]
+  --help                      Show this message and exit.
+
+
+Command: show-cfg
+ Usage:  [OPTIONS]
+
+  Show the current configuration context
+
+Options:
+  --cfg-file PATH  Path to the config file
+  -pp              Pretty Print
+  --help           Show this message and exit.
+
+
+Command: sniff
+ Usage:  [OPTIONS] [TCPDUMP-ARGS] [DEVICE]
+
+  Sniff packets with tcpdump
+
+Options:
+  Testbed sources: 
+    --db, --database TEXT  Database of device. Only needed if not current
+                           default.  [env var: IOTTB_DB]
+    --app TEXT             Companion app being used during capture
+  Runtime behaviour: 
+    --unsafe               Disable checks for otherwise required options.
+                           [env var: IOTTB_UNSAFE]
+    --guided               [env var: IOTTB_GUIDED]
+    --pre PATH             Script to be executed before main commandis
+                           started.
+  Tcpdump options: 
+    -i, --interface TEXT   Network interface to capture on.If not specified
+                           tcpdump tries to find and appropriate one.  [env
+                           var: IOTTB_CAPTURE_INTERFACE]
+    -a, --address TEXT     IP or MAC address to filter packets by.  [env var:
+                           IOTTB_CAPTURE_ADDRESS]
+    -I, --monitor-mode     Put interface into monitor mode.
+    --ff TEXT              tcpdump filter as string or file path.  [env var:
+                           IOTTB_CAPTURE_FILTER]
+    -#, --print-pacno      Print packet number at beginning of line. True by
+                           default.
+    -e, --print-ll         Print link layer headers. True by default.
+    -c, --count INTEGER    Number of packets to capture.
+  --help                   Show this message and exit.
+
+
+Command: show-all
+ Usage:  [OPTIONS]
+
+  Show everything: configuration, databases, and device metadata
+
+Options:
+  --help  Show this message and exit.
+
+

code/iottb-project/iottb/__init__.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+from iottb import definitions
+import logging
+from iottb.utils.user_interaction import tb_echo
+import click
+
+click.echo = tb_echo  # This is very hacky
+logging.basicConfig(level=definitions.LOGLEVEL)
+log_dir = definitions.LOGDIR
+# Ensure logs dir exists before new handlers are registered in main.py
+if not log_dir.is_dir():
+    log_dir.mkdir()
+
+DOCS_FOLDER = Path.cwd() / 'docs'
+

code/iottb-project/iottb/commands/add_device.py

@@ -0,0 +1,89 @@
+import json
+
+import click
+from pathlib import Path
+import logging
+import re
+
+from iottb import definitions
+from iottb.models.device_metadata import DeviceMetadata
+from iottb.models.iottb_config import IottbConfig
+from iottb.definitions import CFG_FILE_PATH, TB_ECHO_STYLES
+
+logger = logging.getLogger(__name__)
+
+
+def add_device_guided(ctx, cn, db):
+    click.echo('TODO: Implement')
+    logger.info('Adding device interactively')
+    #logger.debug(f'Parameters: {params}. value: {value}')
+
+
+@click.command('add-device', help='Add a device to a database')
+@click.option('--dev', '--device-name', type=str, required=True,
+              help='The name of the device to be added. If this string contains spaces or other special characters \
+              normalization is performed to derive a canonical name')
+@click.option('--db', '--database', type=click.Path(exists=True, file_okay=False, dir_okay=True),
+              envvar='IOTTB_DB', show_envvar=True,
+              help='Database in which to add this device. If not specified use default from config.')
+@click.option('--guided', is_flag=True, default=False, show_default=True, envvar='IOTTB_GUIDED_ADD', show_envvar=True,
+              help='Add device interactively')
+def add_device(dev, db, guided):
+    """Add a new device to a database
+
+        Device name must be supplied unless in an interactive setup. Database is taken from config by default.
+    """
+    logger.info('add-device invoked')
+
+    # Step 1: Load Config
+    # Dependency: Config file must exist
+    config = IottbConfig(Path(CFG_FILE_PATH))
+    logger.debug(f'Config loaded: {config}')
+
+    # Step 2: Load database
+    # dependency: Database folder must exist
+    if db:
+        database = db
+        path = config.db_path_dict[database]
+        logger.debug(f'Resolved (path, db) {path}, {database}')
+    else:
+        path = config.default_db_location
+        database = config.default_database
+        logger.debug(f'Default (path, db) {path}, {database}')
+    click.secho(f'Using database {database}')
+    full_db_path = Path(path) / database
+    if not full_db_path.is_dir():
+        logger.warning(f'No database at {database}')
+        click.echo(f'Could not find a database.')
+        click.echo(f'You need to initialize the testbed before before you add devices!')
+        click.echo(f'To initialize the testbed in the default location run "iottb init-db"')
+        click.echo('Exiting...')
+        exit()
+
+    # Step 3: Check if device already exists in database
+    # dependency: DeviceMetadata object
+    device_metadata = DeviceMetadata(device_name=dev)
+    device_dir = full_db_path / device_metadata.canonical_name
+
+    # Check if device is already registered
+    if device_dir.exists():
+        logger.warning(f'Device directory {device_dir} already exists.')
+        click.echo(f'Device {dev} already exists in the database.')
+        click.echo('Exiting...')
+        exit()
+    try:
+        device_dir.mkdir()
+    except OSError as e:
+        logger.error(f'Error trying to create device {e}')
+        click.echo('Exiting...')
+        exit()
+
+    # Step 4: Save metadata into device_dir
+    metadata_path = device_dir / definitions.DEVICE_METADATA_FILE_NAME
+    with metadata_path.open('w') as metadata_file:
+        json.dump(device_metadata.__dict__, metadata_file, indent=4)
+    click.echo(f'Successfully added device {dev} to database')
+    logger.debug(f'Added device {dev} to database {database}. Full path of metadata {metadata_path}')
+    logger.info(f'Metadata for {dev} {device_metadata.print_attributes()}')
+
+

code/iottb-project/iottb/commands/developer.py

@@ -0,0 +1,130 @@
+from pathlib import Path
+import logging
+import click
+
+from iottb import tb_echo
+from iottb.definitions import DB_NAME, CFG_FILE_PATH
+from iottb.models.iottb_config import IottbConfig
+
+logger = logging.getLogger(__name__)
+
+
+@click.group('util')
+def tb():
+    pass
+
+
+@click.command()
+@click.option('--file', default=DB_NAME)
+@click.option('--table', type=str, default='DefaultDatabase')
+@click.option('--key')
+@click.option('--value')
+@click.pass_context
+def set_key_in_table_to(ctx, file, table, key, value):
+    """Edit config or metadata files. TODO: Implement"""
+    click.echo(f'set_key_in_table_to invoked')
+    logger.warning("Unimplemented subcommand invoked.")
+
+
+@click.command()
+@click.confirmation_option(prompt="Are you certain that you want to delete the cfg file?")
+def rm_cfg():
+    """ Removes the cfg file from the filesystem.
+
+        This is mostly a utility during development. Once non-standard database locations are implemented,
+        deleting this would lead to iottb not being able to find them anymore.
+    """
+    Path(CFG_FILE_PATH).unlink()
+    click.echo(f'Iottb configuration removed at {CFG_FILE_PATH}')
+
+
+@click.command()
+@click.confirmation_option(prompt="Are you certain that you want to delete the databases file?")
+def rm_dbs(dbs):
+    """ Removes ALL(!) databases from the filesystem if they're empty.
+
+        Development utility currently unfit for use.
+    """
+    config = IottbConfig()
+    paths = config.get_know_database_paths()
+    logger.debug(f'Known db paths: {str(paths)}')
+    for dbs in paths:
+        try:
+            Path(dbs).rmdir()
+            click.echo(f'{dbs} deleted')
+        except Exception as e:
+            logger.debug(f'Failed unlinking db {dbs} with error {e}')
+    logger.info(f'All databases deleted')
+
+
+@click.command('show-cfg', help='Show the current configuration context')
+@click.option('--cfg-file', type=click.Path(), default=CFG_FILE_PATH, help='Path to the config file')
+@click.option('-pp', is_flag=True, default=False, help='Pretty Print')
+@click.pass_context
+def show_cfg(ctx, cfg_file, pp):
+    logger.debug(f'Pretty print option set to {pp}')
+    if pp:
+        try:
+            config = IottbConfig(Path(cfg_file))
+            click.echo("Configuration Context:")
+            click.echo(f"Default Database: {config.default_database}")
+            click.echo(f"Default Database Path: {config.default_db_location}")
+            click.echo("Database Locations:")
+            for db_name, db_path in config.db_path_dict.items():
+                click.echo(f"  - {db_name}: {db_path}")
+        except Exception as e:
+            logger.error(f"Error loading configuration: {e}")
+            click.echo(f"Failed to load configuration from {cfg_file}")
+    else:
+        path = Path(cfg_file)
+
+        if path.is_file():
+            with path.open('r') as file:
+                content = file.read()
+                click.echo(content)
+        else:
+            click.echo(f"Configuration file not found at {cfg_file}")
+
+
+@click.command('show-all', help='Show everything: configuration, databases, and device metadata')
+@click.pass_context
+def show_everything(ctx):
+    """Show everything that can be recursively found based on config except file contents."""
+    config = ctx.obj['CONFIG']
+    click.echo("Configuration Context:")
+    click.echo(f"Default Database: {config.default_database}")
+    click.echo(f"Default Database Path: {config.default_db_location}")
+    click.echo("Database Locations:")
+    everything_dict = {}
+    for db_name, db_path in config.db_path_dict.items():
+
+        click.echo(f"  - {db_name}: {db_path}")
+    for db_name, db_path in config.db_path_dict.items():
+        full_db_path = Path(db_path) / db_name
+        if full_db_path.is_dir():
+            click.echo(f"\nContents of {full_db_path}:")
+            flag = True
+            for item in full_db_path.iterdir():
+                flag = False
+                if item.is_file():
+                    click.echo(f"    - {item.name}")
+                    try:
+                        with item.open('r', encoding='utf-8') as file:
+                            content = file.read()
+                            click.echo(f"      Content:\n{content}")
+                    except UnicodeDecodeError:
+                        click.echo("      Content is not readable as text")
+                elif item.is_dir():
+                    click.echo(f"    - {item.name}/")
+                    for subitem in item.iterdir():
+                        if subitem.is_file():
+                            click.echo(f"      - {subitem.name}")
+                        elif subitem.is_dir():
+                            click.echo(f"      - {subitem.name}/")
+            if flag:
+                tb_echo(f'\t EMPTY')
+        else:
+            click.echo(f"{full_db_path} is not a directory")
+
+
+

code/iottb-project/iottb/commands/sniff.py

@@ -0,0 +1,346 @@
+import json
+import logging
+import os
+import re
+import subprocess
+import uuid
+from datetime import datetime
+from pathlib import Path
+from time import time
+
+import click
+from click_option_group import optgroup
+
+from iottb.utils.string_processing import make_canonical_name
+
+# Setup logger
+logger = logging.getLogger('iottb.sniff')
+
+
+def is_ip_address(address):
+    ip_pattern = re.compile(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$")
+    return ip_pattern.match(address) is not None
+
+
+def is_mac_address(address):
+    mac_pattern = re.compile(r"^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$")
+    return mac_pattern.match(address) is not None
+
+
+def load_config(cfg_file):
+    """Loads configuration from the given file path."""
+    with open(cfg_file, 'r') as config_file:
+        return json.load(config_file)
+
+
+def validate_sniff(ctx, param, value):
+    logger.info('Validating sniff...')
+    if ctx.params.get('unsafe') and not value:
+        return None
+    if not ctx.params.get('unsafe') and not value:
+        raise click.BadParameter('Address is required unless --unsafe is set.')
+    if not is_ip_address(value) and not is_mac_address(value):
+        raise click.BadParameter('Address must be a valid IP address or MAC address.')
+    return value
+
+
+def run_pre(pre):
+    subprocess.run(pre, shell=True)
+    logger.debug(f'finnished {pre}')
+
+
+def run_post(post):
+    subprocess.run(post, shell=True)
+    logger.debug(f'finnished {post}')
+
+
+@click.command('sniff', help='Sniff packets with tcpdump')
+@optgroup.group('Testbed sources')
+@optgroup.option('--db', '--database', type=str, envvar='IOTTB_DB', show_envvar=True,
+                 help='Database of device. Only needed if not current default.')
+@optgroup.option('--app', type=str, help='Companion app being used during capture', required=False)
+@optgroup.group('Runtime behaviour')
+@optgroup.option('--unsafe', is_flag=True, default=False, envvar='IOTTB_UNSAFE', is_eager=True,
+                 help='Disable checks for otherwise required options.\n', show_envvar=True)
+@optgroup.option('--guided', is_flag=True, default=False, envvar='IOTTB_GUIDED', show_envvar=True)
+@optgroup.option('--pre', help='Script to be executed before main  command is started.')
+@optgroup.option('--post',  help='Script to be executed upon completion of main  command.')
+@optgroup.group('Tcpdump options')
+@optgroup.option('-i', '--interface',
+                 help='Network interface to capture on.' +
+                      'If not specified tcpdump tries to find and appropriate one.\n', show_envvar=True,
+                 envvar='IOTTB_CAPTURE_INTERFACE')
+@optgroup.option('-a', '--address', callback=validate_sniff,
+                 help='IP or MAC address to filter packets by.\n', show_envvar=True,
+                 envvar='IOTTB_CAPTURE_ADDRESS')
+@optgroup.option('-I', '--monitor-mode', help='Put interface into monitor mode.\n', is_flag=True)
+@optgroup.option('--ff', type=str, envvar='IOTTB_CAPTURE_FILTER', show_envvar=True,
+                 help='tcpdump filter as string or file path.')
+@optgroup.option('-#', '--print-pacno', is_flag=True, default=True,
+                 help='Print packet number at beginning of line. True by default.\n')
+@optgroup.option('-e', '--print-ll', is_flag=True, default=False,
+                 help='Print link layer headers. True by default.')
+@optgroup.option('-c', '--count', type=int, help='Number of packets to capture.', default=1000)
+# @optgroup.option('--mins', type=int, help='Time in minutes to capture.', default=1)
+@click.argument('tcpdump-args', nargs=-1, required=False, metavar='[TCPDUMP-ARGS]')
+@click.argument('device', required=False)
+@click.pass_context
+def sniff(ctx, device, interface, print_pacno, ff, count, monitor_mode, print_ll, address, db, unsafe, guided,
+          app, tcpdump_args, pre, post, **params):
+    """ Sniff packets from a device """
+    logger.info('sniff command invoked')
+    # Step 0: run pre script:
+    if pre:
+        click.echo(f'Running pre command {pre}')
+        run_pre(pre)
+    # Step1: Load Config
+    config = ctx.obj['CONFIG']
+    logger.debug(f'Config loaded: {config}')
+
+    # Step2: determine relevant database
+    database = db if db else config.default_database
+    path = config.db_path_dict[database]
+    full_db_path = Path(path) / database
+    logger.debug(f'Full db path is {str(full_db_path)}')
+
+    # 2.2: Check if it exists
+    if not full_db_path.is_dir():
+        logger.error('DB unexpectedly missing')
+        click.echo('DB unexpectedly missing')
+        return
+
+    canonical_name, aliases = make_canonical_name(device)
+    click.echo(f'Using canonical device name {canonical_name}')
+    device_path = full_db_path / canonical_name
+
+    # Step 3: now the device
+    if not device_path.exists():
+        if not unsafe:
+            logger.error(f'Device path {device_path} does not exist')
+            click.echo(f'Device path {device_path} does not exist')
+            return
+        else:
+            device_path.mkdir(parents=True, exist_ok=True)
+            logger.info(f'Device path {device_path} created')
+
+    click.echo(f'Found device at path {device_path}')
+    # Step 4: Generate filter
+    generic_filter = None
+    cap_filter = None
+    if ff:
+        logger.debug(f'ff: {ff}')
+        if Path(ff).is_file():
+            logger.info('Given filter option is a file')
+            with open(ff, 'r') as f:
+                cap_filter = f.read().strip()
+        else:
+            logger.info('Given filter option is an expression')
+            cap_filter = ff
+    else:
+        if address is not None:
+            if is_ip_address(address):
+                generic_filter = 'net'
+                cap_filter = f'{generic_filter} {address}'
+            elif is_mac_address(address):
+                generic_filter = 'ether net'
+                cap_filter = f'{generic_filter} {address}'
+            elif not unsafe:
+                logger.error('Invalid address format')
+                click.echo('Invalid address format')
+                return
+
+    logger.info(f'Generic filter {generic_filter}')
+    click.echo(f'Using filter {cap_filter}')
+
+    # Step 5: prep capture directory
+    capture_date = datetime.now().strftime('%Y-%m-%d')
+    capture_base_dir = device_path / f'sniffs/{capture_date}'
+    capture_base_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.debug(f'Previous captures {capture_base_dir.glob('cap*')}')
+    capture_count = sum(1 for _ in capture_base_dir.glob('cap*'))
+    logger.debug(f'Capture count is {capture_count}')
+
+    capture_dir = f'cap{capture_count:04d}-{datetime.now().strftime('%H%M')}'
+    logger.debug(f'capture_dir: {capture_dir}')
+
+    # Full path
+    capture_dir_full_path = capture_base_dir / capture_dir
+    capture_dir_full_path.mkdir(parents=True, exist_ok=True)
+
+    click.echo(f'Files will be placed in {str(capture_dir_full_path)}')
+    logger.debug(f'successfully created capture directory')
+
+    # Step 6: Prepare capture file names
+    # Generate UUID for filenames
+    capture_uuid = str(uuid.uuid4())
+    click.echo(f'Capture has id {capture_uuid}')
+
+    pcap_file = f"{canonical_name}_{capture_uuid}.pcap"
+    pcap_file_full_path = capture_dir_full_path / pcap_file
+    stdout_log_file = f'stdout_{capture_uuid}.log'
+    stderr_log_file = f'stderr_{capture_uuid}.log'
+
+    logger.debug(f'Full pcap file path is {pcap_file_full_path}')
+    logger.info(f'pcap file name is {pcap_file}')
+    logger.info(f'stdout log file is {stdout_log_file}')
+    logger.info(f'stderr log file is {stderr_log_file}')
+
+    # Step 7: Build tcpdump command
+    logger.debug(f'pgid {os.getpgrp()}')
+    logger.debug(f'ppid {os.getppid()}')
+    logger.debug(f'(real, effective, saved) user id: {os.getresuid()}')
+    logger.debug(f'(real, effective, saved) group id: {os.getresgid()}')
+
+    cmd = ['sudo', 'tcpdump']
+
+    # 7.1 process flags
+    flags = []
+    if print_pacno:
+        flags.append('-#')
+    if print_ll:
+        flags.append('-e')
+    if monitor_mode:
+        flags.append('-I')
+    flags.append('-n')  # TODO: Integrate, in case name resolution is wanted!
+    cmd.extend(flags)
+    flags_string = " ".join(flags)
+    logger.debug(f'Flags: {flags_string}')
+
+    # debug interlude
+    verbosity = ctx.obj['VERBOSITY']
+    if verbosity > 0:
+        verbosity_flag = '-'
+        for i in range(0, verbosity):
+            verbosity_flag = verbosity_flag + 'v'
+        logger.debug(f'verbosity string to pass to tcpdump: {verbosity_flag}')
+        cmd.append(verbosity_flag)
+
+    # 7.2 generic (i.e. reusable) kw args
+    generic_kw_args = []
+    if count:
+        generic_kw_args.extend(['-c', str(count)])
+    # if mins:
+    #     generic_kw_args.extend(['-G', str(mins * 60)]) TODO: this currently loads to errors with sudo
+    cmd.extend(generic_kw_args)
+    generic_kw_args_string = " ".join(generic_kw_args)
+    logger.debug(f'KW args: {generic_kw_args_string}')
+
+    # 7.3 special kw args (not a priori reusable)
+    non_generic_kw_args = []
+    if interface:
+        non_generic_kw_args.extend(['-i', interface])
+    non_generic_kw_args.extend(['-w', str(pcap_file_full_path)])
+    cmd.extend(non_generic_kw_args)
+    non_generic_kw_args_string = " ".join(non_generic_kw_args)
+    logger.debug(f'Non transferable (special) kw args: {non_generic_kw_args_string}')
+
+    # 7.4 add filter expression
+    if cap_filter:
+        logger.debug(f'cap_filter (not generic): {cap_filter}')
+        cmd.append(cap_filter)
+
+    full_cmd_string = " ".join(cmd)
+
+    logger.info(f'tcpdump command: {"".join(full_cmd_string)}')
+    click.echo('Capture setup complete!')
+    # Step 8: Execute tcpdump command
+    start_time = datetime.now().strftime("%H:%M:%S")
+    start = time()
+    try:
+        if guided:
+            click.confirm(f'Execute following command: {full_cmd_string}')
+        stdout_log_file_abs_path = capture_dir_full_path / stdout_log_file
+        stderr_log_file_abs_path = capture_dir_full_path / stderr_log_file
+        stdout_log_file_abs_path.touch(mode=0o777)
+        stderr_log_file_abs_path.touch(mode=0o777)
+        with open(stdout_log_file_abs_path, 'w') as out, open(stderr_log_file_abs_path, 'w') as err:
+            logger.debug(f'\nstdout: {out}.\nstderr: {err}.\n')
+
+            tcp_complete = subprocess.run(cmd, check=True, capture_output=True, text=True)
+
+            out.write(tcp_complete.stdout)
+            err.write(tcp_complete.stderr)
+
+            # click.echo(f'Mock sniff execution')
+        click.echo(f"Capture complete. Saved to {pcap_file}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f'Failed to capture packets: {e}')
+        click.echo(f'Failed to capture packets: {e}')
+        click.echo(f'Check {stderr_log_file} for more info.')
+        if ctx.obj['DEBUG']:
+            msg = [f'STDERR log {stderr_log_file} contents:\n']
+            with open(capture_dir_full_path / stderr_log_file) as log:
+                for line in log:
+                    msg.append(line)
+
+            click.echo("\t".join(msg), lvl='e')
+            # print('DEBUG ACTIVE')
+        if guided:
+            click.prompt('Create metadata anyway?')
+        else:
+            click.echo('Aborting capture...')
+            exit()
+    end_time = datetime.now().strftime("%H:%M:%S")
+    end = time()
+    delta = end - start
+
+
+    click.echo(f'tcpdump took {delta:.2f} seconds.')
+    # Step 9: Register metadata
+    metadata = {
+        'device': canonical_name,
+        'device_id': device,
+        'capture_id': capture_uuid,
+        'capture_date_iso': datetime.now().isoformat(),
+        'invoked_command': " ".join(map(str, cmd)),
+        'capture_duration': delta,
+        'generic_parameters': {
+            'flags': flags_string,
+            'kwargs': generic_kw_args_string,
+            'filter': generic_filter
+        },
+        'non_generic_parameters': {
+            'kwargs': non_generic_kw_args_string,
+            'filter': cap_filter
+        },
+        'features': {
+            'interface': interface,
+            'address': address
+        },
+        'resources': {
+            'pcap_file': str(pcap_file),
+            'stdout_log': str(stdout_log_file),
+            'stderr_log': str(stderr_log_file),
+            'pre': str(pre),
+            'post': str(post)
+        },
+        'environment': {
+            'capture_dir': capture_dir,
+            'database': database,
+            'capture_base_dir': str(capture_base_dir),
+            'capture_dir_abs_path': str(capture_dir_full_path)
+        }
+    }
+
+    click.echo('Ensuring correct ownership of created files.')
+    username = os.getlogin()
+    gid = os.getgid()
+
+    # Else there are issues when running with sudo:
+    try:
+        subprocess.run(f'sudo chown -R {username}:{username} {device_path}', shell=True)
+    except OSError as e:
+        click.echo(f'Some error {e}')
+
+    click.echo(f'Saving metadata.')
+    metadata_abs_path = capture_dir_full_path / 'capture_metadata.json'
+    with open(metadata_abs_path, 'w') as f:
+        json.dump(metadata, f, indent=4)
+
+    click.echo(f'END  SNIFF SUBCOMMAND')
+    if post:
+        click.echo(f'Running post script {post}')
+        run_post(post)
+
+

code/iottb-project/iottb/commands/testbed.py

@@ -0,0 +1,67 @@
+import click
+from pathlib import Path
+import logging
+from logging.handlers import RotatingFileHandler
+import sys
+from iottb.models.iottb_config import IottbConfig
+from iottb.definitions import DB_NAME, CFG_FILE_PATH
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option('-d', '--dest', type=click.Path(), help='Location to put (new) iottb database')
+@click.option('-n', '--name', default=DB_NAME, type=str, help='Name of new database.')
+@click.option('--update-default/--no-update-default', default=True, help='If new db should be set as the new default')
+@click.pass_context
+def init_db(ctx, dest, name, update_default):
+    logger.info('init-db invoked')
+    config = ctx.obj['CONFIG']
+    logger.debug(f'str(config)')
+    # Use the default path from config if dest is not provided
+    known_dbs = config.get_known_databases()
+    logger.debug(f'Known databases: {known_dbs}')
+    if name in known_dbs:
+        dest = config.get_database_location(name)
+        if Path(dest).joinpath(name).is_dir():
+            click.echo(f'A database {name} already exists.')
+            logger.debug(f'DB {name} exists in {dest}')
+            click.echo(f'Exiting...')
+            exit()
+        logger.debug(f'DB name {name} registered but does not exist.')
+    if not dest:
+        logger.info('No dest set, choosing default destination.')
+        dest = Path(config.default_db_location).parent
+
+    db_path = Path(dest).joinpath(name)
+    logger.debug(f'Full path for db {str(db_path)}')
+    # Create the directory if it doesn't exist
+    db_path.mkdir(parents=True, exist_ok=True)
+    logger.info(f"mkdir {db_path} successful")
+    click.echo(f'Created {db_path}')
+
+    # Update configuration
+    config.set_database_location(name, str(dest))
+    if update_default:
+        config.set_default_database(name, str(dest))
+    config.save_config()
+    logger.info(f"Updated configuration with database {name} at {db_path}")
+
+
+# @click.group('config')
+# @click.pass_context
+# def cfg(ctx):
+#     pass
+#
+# @click.command('set', help='Set the location of a database.')
+# @click.argument('database', help='Name of database')
+# @click.argument('location', help='Where the database is located (i.e. its parent directory)')
+# @click.pass_context
+# def set(ctx, key, value):
+#     click.echo(f'Setting {key} to {value} in config')
+#     config = ctx.obj['CONFIG']
+#     logger.warning('No checks performed!')
+#     config.set_database_location(key, value)
+#     config.save_config()
+
+

code/iottb-project/iottb/definitions.py

@@ -0,0 +1,48 @@
+import logging
+from pathlib import Path
+
+import click
+
+APP_NAME = 'iottb'
+DB_NAME = 'iottb.db'
+CFG_FILE_PATH = str(Path(click.get_app_dir(APP_NAME)).joinpath('iottb.cfg'))
+CONSOLE_LOG_FORMATS = {
+    0: '%(levelname)s - %(message)s',
+    1: '%(levelname)s - %(module)s - %(message)s',
+    2: '%(levelname)s - %(module)s - %(funcName)s - %(lineno)d - %(message)s'
+}
+
+LOGFILE_LOG_FORMAT = {
+    0: '%(levelname)s - %(asctime)s - %(module)s  - %(message)s',
+    1: '%(levelname)s - %(asctime)s - %(module)s - %(funcName)s - %(message)s',
+    2: '%(levelname)s - %(asctime)s - %(module)s - %(funcName)s - %(lineno)d - %(message)s'
+}
+MAX_VERBOSITY = len(CONSOLE_LOG_FORMATS) - 1
+assert len(LOGFILE_LOG_FORMAT) == len(CONSOLE_LOG_FORMATS), 'Log formats must be same size'
+
+LOGLEVEL = logging.DEBUG
+LOGDIR = Path.cwd() / 'logs'
+
+# Characters to just replace
+REPLACEMENT_SET_CANONICAL_DEVICE_NAMES = {' ', '_', ',', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '+', '=',
+                                          '{', '}', '[', ']',
+                                          '|',
+                                          '\\', ':', ';', '"', "'", '<', '>', '?', '/', '`', '~'}
+# Characters to possibly error on
+ERROR_SET_CANONICAL_DEVICE_NAMES = {',', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '+', '=', '{', '}', '[', ']',
+                                    '|',
+                                    '\\', ':', ';', '"', "'", '<', '>', '?', '/', '`', '~'}
+
+DEVICE_METADATA_FILE_NAME = 'device_metadata.json'
+
+TB_ECHO_STYLES = {
+    'w': {'fg': 'yellow', 'bold': True},
+    'i': {'fg': 'blue', 'italic': True},
+    's': {'fg': 'green', 'bold': True},
+    'e': {'fg': 'red', 'bold': True},
+    'header': {'fg': 'bright_cyan', 'bold': True, 'italic': True}
+}
+
+NAME_OF_CAPTURE_DIR = 'sniffs'
+
+

code/iottb-project/iottb/main.py

@@ -0,0 +1,77 @@
+import os
+import shutil
+
+import click
+from pathlib import Path
+import logging
+
+from iottb.commands.sniff import sniff
+from iottb.commands.developer import set_key_in_table_to, rm_cfg, rm_dbs, show_cfg, show_everything
+
+##################################################
+# Import package modules
+#################################################
+from iottb.utils.logger_config import setup_logging
+from iottb import definitions
+from iottb.models.iottb_config import IottbConfig
+from iottb.commands.testbed import init_db
+from iottb.commands.add_device import add_device
+
+############################################################################
+# Module shortcuts for global definitions
+###########################################################################
+APP_NAME = definitions.APP_NAME
+DB_NAME = definitions.DB_NAME
+CFG_FILE_PATH = definitions.CFG_FILE_PATH
+# These are (possibly) redundant when defined in definitions.py
+# keeping them here until refactored and tested
+MAX_VERBOSITY = definitions.MAX_VERBOSITY
+
+# Logger stuff
+loglevel = definitions.LOGLEVEL
+logger = logging.getLogger(__name__)
+
+
+@click.group(context_settings=dict(auto_envvar_prefix='IOTTB', show_default=True))
+@click.option('-v', '--verbosity', count=True, type=click.IntRange(0, 3), default=0, is_eager=True,
+              help='Set verbosity')
+@click.option('-d', '--debug', is_flag=True, default=False, is_eager=True,
+              help='Enable debug mode')
+@click.option('--dry-run', is_flag=True, default=True, is_eager=True)
+@click.option('--cfg-file', type=click.Path(),
+              default=Path(click.get_app_dir(APP_NAME)).joinpath('iottb.cfg'),
+              envvar='IOTTB_CONF_HOME', help='Path to iottb config file')
+@click.pass_context
+def cli(ctx, verbosity, debug, dry_run, cfg_file):
+    setup_logging(verbosity, debug)  # Setup logging based on the loaded configuration and other options
+    ctx.ensure_object(dict)  # Make sure context is ready for use
+    logger.info("Starting execution.")
+    ctx.obj['CONFIG'] = IottbConfig(cfg_file)  # Load configuration directly
+    ctx.meta['FULL_PATH_CONFIG_FILE'] = str(cfg_file)
+    ctx.meta['DRY_RUN'] = dry_run
+    logger.debug(f'Verbosity: {verbosity}')
+    ctx.obj['VERBOSITY'] = verbosity
+    logger.debug(f'Debug: {debug}')
+    ctx.obj['DEBUG'] = debug
+
+
+##################################################################################
+# Add all subcommands to group here
+#################################################################################
+# TODO: Is there a way to do this without pylint freaking out?
+# noinspection PyTypeChecker
+cli.add_command(init_db)
+cli.add_command(rm_cfg)
+cli.add_command(set_key_in_table_to)
+cli.add_command(rm_dbs)
+# noinspection PyTypeChecker
+cli.add_command(add_device)
+cli.add_command(show_cfg)
+cli.add_command(sniff)
+cli.add_command(show_everything)
+
+
+if __name__ == '__main__':
+    cli()
+    for log in Path.cwd().iterdir():
+        log.chmod(0o777)

code/iottb-project/iottb/models/database.py

@@ -0,0 +1,6 @@
+class Database:
+
+    def __init__(self, name, path):
+        self.name = name
+        self.path = path
+        self.device_list = [] # List of the canonical names of devices registered in this database

code/iottb-project/iottb/models/device_metadata.py

@@ -0,0 +1,44 @@
+import logging
+import uuid
+from datetime import datetime
+import logging
+import click
+
+from iottb.utils.string_processing import make_canonical_name
+
+logger = logging.getLogger(__name__)
+
+
+class DeviceMetadata:
+    def __init__(self, device_name, description="", model="", manufacturer="", firmware_version="", device_type="",
+                 supported_interfaces="", companion_applications="", save_to_file=None):
+        self.device_id = str(uuid.uuid4())
+        self.device_name = device_name
+        cn, aliases = make_canonical_name(device_name)
+        logger.debug(f'cn, aliases = {cn}, {str(aliases)}')
+        self.aliases = aliases
+        self.canonical_name = cn
+        self.date_added = datetime.now().isoformat()
+        self.description = description
+        self.model = model
+        self.manufacturer = manufacturer
+        self.current_firmware_version = firmware_version
+        self.device_type = device_type
+        self.supported_interfaces = supported_interfaces
+        self.companion_applications = companion_applications
+        self.last_metadata_update = datetime.now().isoformat()
+        if save_to_file is not None:
+            click.echo('TODO: Implement saving config to file after creation!')
+
+    def add_alias(self, alias: str = ""):
+        if alias == "":
+            return
+        self.aliases.append(alias)
+
+    def get_canonical_name(self):
+        return self.canonical_name
+
+    def print_attributes(self):
+        print(f'Printing attribute value pairs in {__name__}')
+        for attr, value in self.__dict__.items():
+            print(f'{attr}: {value}')

code/iottb-project/iottb/models/iottb_config.py

@@ -0,0 +1,124 @@
+import json
+from pathlib import Path
+
+from iottb import definitions
+import logging
+
+logger = logging.getLogger(__name__)
+
+DB_NAME = definitions.DB_NAME
+
+
+class IottbConfig:
+    """ Class to handle testbed configuration.
+
+       TODO: Add instead of overwrite Database locations when initializing if a location with valid db
+        exists.
+    """
+
+    @staticmethod
+    def warn():
+        logger.warning(f'DatabaseLocations are DatabaseLocationMap in the class {__name__}')
+
+    def __init__(self, cfg_file=definitions.CFG_FILE_PATH):
+        logger.info('Initializing Config object')
+        IottbConfig.warn()
+        self.cfg_file = Path(cfg_file)
+        self.default_database = None
+        self.default_db_location = None
+        self.db_path_dict = dict()
+        self.load_config()
+
+    def create_default_config(self):
+        """Create default iottb config file."""
+        logger.info(f'Creating default config file at {self.cfg_file}')
+        self.default_database = DB_NAME
+        self.default_db_location = str(Path.home())
+        self.db_path_dict = {
+            DB_NAME: self.default_db_location
+        }
+
+        defaults = {
+            'DefaultDatabase': self.default_database,
+            'DefaultDatabasePath': self.default_db_location,
+            'DatabaseLocations': self.db_path_dict
+        }
+
+        try:
+            self.cfg_file.parent.mkdir(parents=True, exist_ok=True)
+            with self.cfg_file.open('w') as config_file:
+                json.dump(defaults, config_file, indent=4)
+        except IOError as e:
+            logger.error(f"Failed to create default configuration file at {self.cfg_file}: {e}")
+            raise RuntimeError(f"Failed to create configuration file: {e}") from e
+
+    def load_config(self):
+        """Loads or creates default configuration from given file path."""
+        logger.info('Loading configuration file')
+        if not self.cfg_file.is_file():
+            logger.info('Config file does not exist.')
+            self.create_default_config()
+        else:
+            logger.info('Config file exists, opening.')
+            with self.cfg_file.open('r') as config_file:
+                data = json.load(config_file)
+                self.default_database = data.get('DefaultDatabase')
+                self.default_db_location = data.get('DefaultDatabasePath')
+                self.db_path_dict = data.get('DatabaseLocations', {})
+
+    def save_config(self):
+        """Save the current configuration to the config file."""
+        data = {
+            'DefaultDatabase': self.default_database,
+            'DefaultDatabasePath': self.default_db_location,
+            'DatabaseLocations': self.db_path_dict
+        }
+        try:
+            with self.cfg_file.open('w') as config_file:
+                json.dump(data, config_file, indent=4)
+        except IOError as e:
+            logger.error(f"Failed to save configuration file at {self.cfg_file}: {e}")
+            raise RuntimeError(f"Failed to save configuration file: {e}") from e
+
+    def set_default_database(self, name, path):
+        """Set the default database and its path."""
+        self.default_database = name
+        self.default_db_location = path
+        self.db_path_dict[name] = path
+
+    def get_default_database_location(self):
+        return self.default_db_location
+
+    def get_default_database(self):
+        return self.default_database
+
+    def get_database_location(self, name):
+        """Get the location of a specific database."""
+        return self.db_path_dict.get(name)
+
+    def set_database_location(self, name, path):
+        """Set the location for a database."""
+        logger.debug(f'Type of "path" parameter {type(path)}')
+        logger.debug(f'String value of "path" parameter {str(path)}')
+        logger.debug(f'Type of "name" parameter {type(name)}')
+        logger.debug(f'String value of "name" parameter {str(name)}')
+        path = Path(path)
+        name = Path(name)
+        logger.debug(f'path:name = {path}:{name}')
+        if path.name == name:
+            path = path.parent
+        self.db_path_dict[str(name)] = str(path)
+
+    def get_known_databases(self):
+        """Get the set of known databases"""
+        logger.info(f'Getting known databases.')
+
+        return self.db_path_dict.keys()
+
+    def get_know_database_paths(self):
+        """Get the paths of all known databases"""
+        logger.info(f'Getting known database paths.')
+        return self.db_path_dict.values()
+
+    def get_full_default_path(self):
+        return Path(self.default_db_location) / self.default_database

code/iottb-project/iottb/models/sniff_metadata.py

@@ -0,0 +1,39 @@
+import json
+import logging
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+logger = logging.getLogger('iottb.sniff')  # Log with sniff subcommand
+
+class CaptureMetadata:
+    def __init__(self, device_id, capture_dir, interface, address, capture_file, tcpdump_command, tcpdump_stdout, tcpdump_stderr, packet_filter, alias):
+        self.base_data = {
+            'device_id': device_id,
+            'capture_id': str(uuid.uuid4()),
+            'capture_date': datetime.now().isoformat(),
+            'capture_dir': str(capture_dir),
+            'capture_file': capture_file,
+            'start_time': "",
+            'stop_time': "",
+            'alias': alias
+        }
+        self.features = {
+            'interface': interface,
+            'device_ip_address': address if address else "No IP Address set",
+            'tcpdump_stdout': str(tcpdump_stdout),
+            'tcpdump_stderr': str(tcpdump_stderr),
+            'packet_filter': packet_filter
+        }
+        self.command = tcpdump_command
+
+    def save_to_file(self):
+        metadata = {
+            'base_data': self.base_data,
+            'features': self.features,
+            'command': self.command
+        }
+        metadata_file_path = Path(self.base_data['capture_dir']) / 'metadata.json'
+        with open(metadata_file_path, 'w') as f:
+            json.dump(metadata, f, indent=4)
+        logger.info(f'Metadata saved to {metadata_file_path}')

code/iottb-project/iottb/scripts/generate_help.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+import click
+from io import StringIO
+import sys
+from iottb import DOCS_FOLDER
+# Import your CLI app here
+from iottb.main import cli
+
+"""Script to generate the help text and write to file.
+
+    Definitely needs better formatting.
+    Script is also not very flexible.
+"""
+
+
+def get_help_text(command):
+    """Get the help text for a given command."""
+    help_text = StringIO()
+    with click.Context(command) as ctx:
+        # chatgpt says this helps: was right
+        sys_stdout = sys.stdout
+        sys.stdout = help_text
+        try:
+            click.echo(command.get_help(ctx))
+        finally:
+            sys.stdout = sys_stdout
+    return help_text.getvalue()
+
+
+def write_help_to_file(cli, filename):
+    """Write help messages of all commands and subcommands to a file."""
+    with open(filename, 'w+') as f:
+        # main
+        f.write(f"Main Command: iottb\n")
+        f.write(get_help_text(cli))
+        f.write("\n\n")
+
+        # go through subcommands
+        for cmd_name, cmd in cli.commands.items():
+            f.write(f"Command: {cmd_name}\n")
+            f.write(get_help_text(cmd))
+            f.write("\n\n")
+
+            # subcommands of subcommands
+            if isinstance(cmd, click.Group):
+                for sub_cmd_name, sub_cmd in cmd.commands.items():
+                    f.write(f"Subcommand: {cmd_name} {sub_cmd_name}\n")
+                    f.write(get_help_text(sub_cmd))
+                    f.write("\n\n")
+
+
+def manual():
+    comands = [
+        'init-db',
+        'add-device',
+        'sniff'
+    ]
+    dev_commands = [
+        'show-all',
+        'rm-dbs',
+        'show-cfg',
+        'show-all'
+    ]
+
+
+if __name__ == "__main__":
+    from iottb import DOCS_FOLDER
+
+    print('Must be in project root for this to work properly!')
+    print(f'CWD is {str(Path.cwd())}')
+    DOCS_FOLDER.mkdir(exist_ok=True)
+    write_help_to_file(cli, str(DOCS_FOLDER / "help_messages.md"))
+    print(f'Wrote help_messages.md to {str(DOCS_FOLDER / "help_messages.md")}')

code/iottb-project/iottb/scripts/sudo_iottb

@@ -0,0 +1,4 @@
+#/bin/sh
+echo 'Running iottb as sudo'
+sudo $(which python) iottb $@
+echo 'Finished executing iottb with sudo'

code/iottb-project/iottb/utils/logger_config.py

@@ -0,0 +1,41 @@
+import logging
+import sys
+from logging.handlers import RotatingFileHandler
+
+from iottb import definitions
+from iottb.definitions import MAX_VERBOSITY, CONSOLE_LOG_FORMATS, APP_NAME, LOGFILE_LOG_FORMAT
+
+loglevel = definitions.LOGLEVEL
+
+
+def setup_logging(verbosity, debug=loglevel):
+    """ Setup root logger for iottb """
+    log_level = loglevel
+    handlers = []
+    date_format = '%Y-%m-%d %H:%M:%S'
+    if verbosity > 0:
+        log_level = logging.WARNING
+        if verbosity > MAX_VERBOSITY:
+            verbosity = MAX_VERBOSITY
+            log_level = logging.INFO
+        assert verbosity <= MAX_VERBOSITY, f'Verbosity must be <= {MAX_VERBOSITY}'
+        console_handler = logging.StreamHandler(sys.stdout)
+        print(str(sys.stdout))
+        console_handler.setFormatter(logging.Formatter(CONSOLE_LOG_FORMATS[verbosity], datefmt=date_format))
+        console_handler.setLevel(logging.DEBUG)  # can keep at debug since it depends on global level?
+        handlers.append(console_handler)
+
+    if debug:
+        log_level = logging.DEBUG
+
+    # Logfile logs INFO+, no debugs though
+    file_handler = RotatingFileHandler(f'{str(definitions.LOGDIR / APP_NAME)}.log', maxBytes=10240, backupCount=5)
+    file_handler.setFormatter(logging.Formatter(LOGFILE_LOG_FORMAT[verbosity], datefmt=date_format))
+    file_handler.setLevel(logging.INFO)
+
+    # finnish root logger setup
+    handlers.append(file_handler)
+    # Force this config to be applied to root logger
+    logging.basicConfig(level=log_level, handlers=handlers, force=True)
+
+

code/iottb-project/iottb/utils/string_processing.py

@@ -0,0 +1,40 @@
+import re
+from iottb import definitions
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def normalize_string(s, chars_to_replace=None, replacement=None, allow_unicode=False):
+    pass
+
+
+def make_canonical_name(name):
+    """
+    Normalize the device name to a canonical form:
+    - Replace the first two occurrences of spaces and transform characters with dashes.
+    - Remove any remaining spaces and non-ASCII characters.
+    - Convert to lowercase.
+    """
+    aliases = [name]
+    logger.info(f'Normalizing name {name}')
+
+    # We first normalize
+    chars_to_replace = definitions.REPLACEMENT_SET_CANONICAL_DEVICE_NAMES
+    pattern = re.compile('|'.join(re.escape(char) for char in chars_to_replace))
+    norm_name = pattern.sub('-', name)
+    norm_name = re.sub(r'[^\x00-\x7F]+', '', norm_name) # removes non ascii chars
+
+    aliases.append(norm_name)
+    # Lower case
+    norm_name = norm_name.lower()
+    aliases.append(norm_name)
+
+    # canonical name is only first two parts of resulting string
+    parts = norm_name.split('-')
+    canonical_name = canonical_name = '-'.join(parts[:2])
+    aliases.append(canonical_name)
+    aliases = list(set(aliases))
+    logger.debug(f'Canonical name: {canonical_name}')
+    logger.debug(f'Aliases: {aliases}')
+    return canonical_name, aliases

code/iottb-project/iottb/utils/user_interaction.py

@@ -0,0 +1,42 @@
+# iottb/utils/user_interaction.py
+
+import click
+from iottb.definitions import TB_ECHO_STYLES
+import sys
+import os
+
+
+def tb_echo2(msg: str, lvl='i', log=True):
+    style = TB_ECHO_STYLES.get(lvl, {})
+    click.secho(f'[IOTTB]', **style)
+    click.secho(f'[IOTTB] \t {msg}', **style)
+
+
+last_prefix = None
+
+
+def tb_echo(msg: str, lvl='i', log=True):
+    global last_prefix
+    prefix = f'Testbed [{lvl.upper()}]\n'
+
+    if last_prefix != prefix:
+        click.secho(prefix, nl=False, **TB_ECHO_STYLES['header'])
+        last_prefix = prefix
+
+    click.secho(f' {msg}', **TB_ECHO_STYLES[lvl])
+
+
+def main():
+    tb_echo('Info message', 'i')
+    tb_echo('Warning message', 'w')
+    tb_echo('Error message', 'e')
+    tb_echo('Success message', 's')
+
+
+if __name__ == '__main__':
+    # arrrgggg hacky
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    project_root = os.path.abspath(os.path.join(current_dir, '../../'))
+    sys.path.insert(0, project_root)
+
+    main()

code/iottb-project/poetry.lock

@@ -0,0 +1,103 @@
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+
+[[package]]
+name = "click"
+version = "8.1.7"
+description = "Composable command line interface toolkit"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"},
+    {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
+[[package]]
+name = "packaging"
+version = "24.1"
+description = "Core utilities for Python packages"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
+    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
+]
+
+[[package]]
+name = "pluggy"
+version = "1.5.0"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]
+
+[[package]]
+name = "pytest"
+version = "8.2.2"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
+    {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=1.5,<2.0"
+
+[package.extras]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
+[[package]]
+name = "scapy"
+version = "2.5.0"
+description = "Scapy: interactive packet manipulation tool"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4"
+files = [
+    {file = "scapy-2.5.0.tar.gz", hash = "sha256:5b260c2b754fd8d409ba83ee7aee294ecdbb2c235f9f78fe90bc11cb6e5debc2"},
+]
+
+[package.extras]
+basic = ["ipython"]
+complete = ["cryptography (>=2.0)", "ipython", "matplotlib", "pyx"]
+docs = ["sphinx (>=3.0.0)", "sphinx_rtd_theme (>=0.4.3)", "tox (>=3.0.0)"]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.12"
+content-hash = "10b2c268b0f10db15eab2cca3d2dc9dc25bc60f4b218ebf786fb780fa85557e0"

code/iottb-project/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "iottb"
+version = "0.1.0"
+description = "IoT Testbed"
+authors = ["Sebastian Lenzlinger <sebastian.lenzlinger@unibas.ch>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+click = "^8.1"
+# scapy = "^2.5"
+
+[tool.poetry.scripts]
+iottb = "iottb.main:cli"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.2.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

code/iottb-project/tests/test_make_canonical_name.py

@@ -0,0 +1,23 @@
+from iottb.utils.string_processing import make_canonical_name
+
+import pytest
+
+
+class TestMakeCanonicalName:
+
+    def test_normalizes_name_with_spaces_to_dashes(self):
+        name = "Device Name With Spaces"
+        expected_canonical_name = "device-name"
+        canonical_name, aliases = make_canonical_name(name)
+        assert canonical_name == expected_canonical_name
+        assert "device-name-with-spaces" in aliases
+        assert "device-name" in aliases
+        assert "Device Name With Spaces" in aliases
+
+    def test_name_with_no_spaces_or_special_characters(self):
+        name = "DeviceName123"
+        expected_canonical_name = "devicename123"
+        canonical_name, aliases = make_canonical_name(name)
+        assert canonical_name == expected_canonical_name
+        assert "DeviceName123" in aliases
+        assert "devicename123" in aliases

code/iottb/__main__.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-from os import environ
-from pathlib import Path
-
-from iottb.logger import logger
-from iottb.subcommands.add_device import setup_init_device_root_parser
-from iottb.subcommands.capture import setup_capture_parser
-from iottb.utils.tcpdump_utils import list_interfaces
-from definitions import IOTTB_HOME_ABS, ReturnCodes
-
-
-######################
-# Argparse setup
-######################
-def setup_argparse():
-    # create top level parser
-    root_parser = argparse.ArgumentParser(prog='iottb')
-    subparsers = root_parser.add_subparsers(title='subcommands', required=True, dest='command')
-
-    # shared options
-    root_parser.add_argument('--verbose', '-v', action='count', default=0)
-    # configure subcommands
-    setup_capture_parser(subparsers)
-    setup_init_device_root_parser(subparsers)
-
-    # Utility to list interfaces directly with iottb instead of relying on external tooling
-
-    interfaces_parser = subparsers.add_parser('list-interfaces', aliases=['li', 'if'],
-                                              help='List available network interfaces.')
-    interfaces_parser.set_defaults(func=list_interfaces)
-
-    return root_parser
-
-
-def check_iottb_env():
-    # This makes the option '--root-dir' obsolescent # TODO How to streamline this?\
-    try:
-        iottb_home = environ['IOTTB_HOME']  # TODO WARN implicit declaration of env var name!
-    except KeyError:
-        logger.error(f"Environment variable 'IOTTB_HOME' is not set."
-                     f"Setting environment variable 'IOTTB_HOME' to '~/{IOTTB_HOME_ABS}'")
-        environ['IOTTB_HOME'] = IOTTB_HOME_ABS
-    finally:
-        if not Path(IOTTB_HOME_ABS).exists():
-            print(f'"{IOTTB_HOME_ABS}" does not exist.')
-            response = input('Do you want to create it now? [y/N]')
-            logger.debug(f'response: {response}')
-            if response.lower() != 'y':
-                logger.debug(f'Not creating "{environ['IOTTB_HOME']}"')
-                print('TODO')
-                print("Aborting execution...")
-                return ReturnCodes.ABORTED
-            else:
-                print(f'Creating "{environ['IOTTB_HOME']}"')
-                Path(IOTTB_HOME_ABS).mkdir(parents=True,
-                                           exist_ok=False)  # Should always work since in 'not exist' code path
-                return ReturnCodes.OK
-        logger.info(f'"{IOTTB_HOME_ABS}" exists.')
-        # TODO: Check that it is a valid iottb dir or can we say it is valid by definition if?
-        return ReturnCodes.OK
-
-
-def main():
-    if check_iottb_env() != ReturnCodes.OK:
-        exit(ReturnCodes.ABORTED)
-    parser = setup_argparse()
-    args = parser.parse_args()
-    print(args)
-    if args.command:
-        try:
-            args.func(args)
-        except KeyboardInterrupt:
-            print('Received keyboard interrupt. Exiting...')
-            exit(1)
-        except Exception as e:
-            print(f'Error: {e}')
-    # create_capture_directory(args.device_name)
-
-
-if __name__ == '__main__':
-    main()

code/iottb/logger.py

@@ -1,28 +0,0 @@
-import logging
-import sys
-from logging.handlers import RotatingFileHandler
-
-
-def setup_logging():
-    logger_obj = logging.getLogger('iottbLogger')
-    logger_obj.setLevel(logging.DEBUG)
-
-    file_handler = RotatingFileHandler('iottb.log')
-    console_handler = logging.StreamHandler(sys.stdout)
-
-    file_handler.setLevel(logging.INFO)
-    console_handler.setLevel(logging.DEBUG)
-
-    file_fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-    console_fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(funcName)s - %(message)s')
-
-    file_handler.setFormatter(file_fmt)
-    console_handler.setFormatter(console_fmt)
-
-    logger_obj.addHandler(file_handler)
-    logger_obj.addHandler(console_handler)
-
-    return logger_obj
-
-
-logger = setup_logging()

notes/journal/2024-05-15-wed.md

@@ -11,4 +11,5 @@ With the above idea it would be possible to also refactor or rewrite how tcpdump
 I want an option such that one can automatically convert a captures resulting file into a csv. Probably will focus on tcpdump for now, since other tools like [[mitmproxy]] have different output files. 
 
 ## Defining Experiment
-I want a pair of commands that 1. provide a guided cli interface to define an experiment and 2. to run that  experiment -> Here [Collective Knowledge Framework](https://github.com/mlcommons/ck) might actually come in handy. The already have tooling for setting up and defining aspects of experiments so that they become reproducible. So maybe one part of the `iottb` as a tool would be to write the correct json files into the directory which contain the informatin on how the command was run. Caveat: All all option values are the same, basically only, if it was used or not (flagging options) or that it was used (e.g. an ip address was used in the filter but the specific value of the ip is of no use for reproducing). Also, Collective Minds tooling relies very common ML algos/framework and static data. So maybe this only comes into play after a capture has been done. So maybe a feature extraction tool (see [[further considerations#Usage paths/ Workflows]]) should create the data and built the database separately.
+I want a pair of commands that 1. provide a guided cli interface to define an experiment and 2. to run that  experiment -> Here [Collective Knowledge Framework](https://github.com/mlcommons/ck) might actually come in handy. The already have tooling for setting up and defining aspects of experiments so that they become reproducible. So maybe one part of the `iottb` as a tool would be to write the correct json files into the directory which contain the informatin on how the command was run. Caveat: All all option values are the same, basically only, if it was used or not (flagging options) or that it was used (e.g. an ip address was used in the filter but the specific value of the ip is of no use for reproducing). Also, Collective Minds tooling relies very common ML algos/framework and static data. So maybe this only comes into play after a capture has been done. So maybe a feature extraction tool (see [[further considerations#Usage paths/ Workflows]]) should create the data and built the database separately.
+#remark TCP dump filter could also be exported into an environment variable? But then again what is the use of defining a conformance, then could use the raw capture idea for tcpdump, too.

notes/scrible

@@ -0,0 +1,13 @@
+`iottb sniff`:
+  min: nothing
+  min meaningfull: interface
+  min usefull: ip/mac addr of dev
+  good: ip/mac, device type
+  better:
+
+`iottb device`
+  `add`: add new device config
+
+`iottb db`
+      `init` initialize device database
+      `add`  add device

notes/scrible.py

@@ -0,0 +1,7 @@
+class Config:
+    db_dir = Path.home()
+    app_config_dir = Path.home /.Config
+    db_name = 'IoTtb.db'
+    app_config_name = 'iottb.conf'
+     
+

thesis/.gitignore

@@ -0,0 +1,33 @@
+*.acn
+*.acr
+*.alg
+*.aux
+*.bbl
+*.blg
+*.dvi
+*.fdb_latexmk
+*.glg
+*.glo
+*.gls
+*.idx
+*.ilg
+*.ind
+*.ist
+*.lof
+*.log
+*.lot
+*.maf
+*.mtc
+*.mtc0
+*.nav
+*.nlo
+*.out
+*.pdfsync
+*.ps
+*.snm
+*.synctex.gz
+*.toc
+*.vrb
+*.xdy
+*.tdo
+*.texpadtmp

thesis/Back/AppendixA.tex

@@ -0,0 +1,162 @@
+% !TEX root = ../Thesis.tex
+\chapter{Appendix A}
+
+\section{Command Line Examples}\label{example:pre-post}
+\subsection{Pre and post scripts}
+In this example, the  \verb|--unsafe| option allows not to specify a IP or MAC address.
+\verb|default| is the device name used and \verb|-c 10| tells \iottb that we only want to capture 10 packets.
+\begin{minted}{bash}
+# Command:
+$ iottb sniff --pre='/usr/bin/echo "pre"' --post='/usr/bin/echo "post"' \ 
+         default --unsafe -c 10
+# Stdout:
+Testbed [Info]
+ Running pre command /usr/bin/echo "pre"
+pre
+ Using canonical device name default
+ Found device at path /home/seb/iottb.db/default
+ Using filter None
+ Files will be placed in /home/seb/iottb.db/default/sniffs/2024-06-30/cap0002-2101
+ Capture has id dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe
+ Capture setup complete!
+ Capture complete. Saved to default_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.pcap
+ tcpdump took 2.12 seconds.
+ Ensuring correct ownership of created files.
+ Saving metadata.
+ END  SNIFF SUBCOMMAND
+ Running post script /usr/bin/echo "post"
+post
+\end{minted}
+
+The contents of the 'sniff' directory for the default device after this capture has completed:
+\begin{minted}{bash}
+sniffs/2024-06-30/cap0002-2101 
+$ tree
+.
+|-- capture_metadata.json
+|-- default_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.pcap
+|-- stderr_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.log
+L__ stdout_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.log
+\end{minted}
+and the metadata file contains (\verb|\| only used for fitting into this document):\\
+\verb|# capture_metadata.json|\\
+\begin{minted}{json}
+{
+"device": "default",
+"device_id": "default",
+"capture_id": "dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe",
+"capture_date_iso": "2024-06-30T21:01:31.496870",
+"invoked_command": "sudo tcpdump -# -n -c 10 -w \
+    /home/seb/iottb.db \
+        /default/sniffs/2024-06-30 \
+            /cap0002-2101/default_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.pcap",
+"capture_duration": 2.117154359817505,
+"generic_parameters": {
+    "flags": "-# -n",
+    "kwargs": "-c 10",
+    "filter": null
+},
+"non_generic_parameters": {
+    "kwargs": "-w \ 
+        /home/seb/iottb.db/default/sniffs/2024-06-30 \ 
+            /cap0002-2101 \ 
+                /default_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.pcap",
+    "filter": null
+},
+"features": {
+    "interface": null,
+    "address": null
+},
+"resources": {
+    "pcap_file": "default_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.pcap",
+    "stdout_log": "stdout_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.log",
+    "stderr_log": "stderr_dcdf1e0b-6c4d-4f01-ba16-f42a04131fbe.log",
+    "pre": "/usr/bin/echo \"pre\"",
+    "post": "/usr/bin/echo \"post\""
+},
+"environment": {
+    "capture_dir": "cap0002-2101",
+    "database": "iottb.db",
+    "capture_base_dir": "/home/seb/iottb.db/default/sniffs/2024-06-30",
+    "capture_dir_abs_path": \
+        "/home/seb/iottb.db/default/sniffs/2024-06-30/cap0002-2101"
+}
+}
+    
+\end{minted}
+
+\section{Canonical Name}
+\begin{listing}[!ht]
+    \inputminted[firstline=12, lastline=40]{python}{string_processing.py}
+    \caption{Shows how the canonical name is created.}
+    \label{lst:dev-canonical}
+\end{listing}
+
+\section{Add Device Example}
+\subsection{Configuration File}\label{appendixA:add-dev-cfg}
+\begin{listing}[!ht]
+\inputminted[linenos, breaklines]{python}{appendixa-after-add-device-dir.txt}
+\caption{Directory and file contents after adding two devices.}
+\label{lst:appendix:appendixa:config-file}
+\end{listing}
+
+\section{Debug Flag Standard Output}
+
+\begin{figure}
+    \centering
+    \begin{minted}{bash}
+❯ iottb -vvv --debug sniff roomba  --unsafe -c 10
+<_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'>
+INFO - main - cli - 48 - Starting execution.
+INFO - iottb_config - __init__ - 24 - Initializing Config object
+WARNING - iottb_config - warn - 21 - DatabaseLocations are DatabaseLocationMap in the class iottb.models.iottb_config
+INFO - iottb_config - load_config - 57 - Loading configuration file
+INFO - iottb_config - load_config - 62 - Config file exists, opening.
+DEBUG - main - cli - 52 - Verbosity: 3
+DEBUG - main - cli - 54 - Debug: True
+INFO - sniff - validate_sniff - 37 - Validating sniff...
+INFO - sniff - sniff - 91 - sniff command invoked
+DEBUG - sniff - sniff - 98 - Config loaded: <iottb.models.iottb_config.IottbConfig object at 0x7f16197d5e50>
+DEBUG - sniff - sniff - 104 - Full db path is /home/seb/showcase
+INFO - string_processing - make_canonical_name - 20 - Normalizing name roomba
+DEBUG - string_processing - make_canonical_name - 38 - Canonical name: roomba
+DEBUG - string_processing - make_canonical_name - 39 - Aliases: ['roomba']
+Testbed [I]
+ Using canonical device name roomba
+ Found device at path /home/seb/showcase/roomba
+INFO - sniff - sniff - 152 - Generic filter None
+ Using filter None
+DEBUG - sniff - sniff - 160 - Previous captures <generator object Path.glob at 0x7f16194ec590>
+DEBUG - sniff - sniff - 162 - Capture count is 4
+DEBUG - sniff - sniff - 165 - capture_dir: cap0004-0310
+ Files will be placed in /home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310
+DEBUG - sniff - sniff - 172 - successfully created capture directory
+ Capture has id 59153b53-c49d-44de-99d2-b5a3490df29a
+DEBUG - sniff - sniff - 185 - Full pcap file path is /home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310/roomba_59153b53-c49d-44de-99d2-b5a3490df29a.pcap
+INFO - sniff - sniff - 186 - pcap file name is roomba_59153b53-c49d-44de-99d2-b5a3490df29a.pcap
+INFO - sniff - sniff - 187 - stdout log file is stdout_59153b53-c49d-44de-99d2-b5a3490df29a.log
+INFO - sniff - sniff - 188 - stderr log file is stderr_59153b53-c49d-44de-99d2-b5a3490df29a.log
+DEBUG - sniff - sniff - 191 - pgid 260696
+DEBUG - sniff - sniff - 192 - ppid 12862
+DEBUG - sniff - sniff - 193 - (real, effective, saved) user id: (1000, 1000, 1000)
+DEBUG - sniff - sniff - 194 - (real, effective, saved) group id: (1000, 1000, 1000)
+DEBUG - sniff - sniff - 209 - Flags: -# -n
+DEBUG - sniff - sniff - 217 - verbosity string to pass to tcpdump: -vvv
+DEBUG - sniff - sniff - 228 - KW args: -c 10
+DEBUG - sniff - sniff - 237 - Non transferable (special) kw args: -w /home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310/roomba_59153b53-c49d-44de-99d2-b5a3490df29a.pcap
+INFO - sniff - sniff - 246 - tcpdump command: sudo tcpdump -# -n -vvv -c 10 -w /home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310/roomba_59153b53-c49d-44de-99d2-b5a3490df29a.pcap
+ Capture setup complete!
+DEBUG - sniff - sniff - 259 - 
+stdout: <_io.TextIOWrapper name='/home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310/stdout_59153b53-c49d-44de-99d2-b5a3490df29a.log' mode='w' encoding='UTF-8'>.
+stderr: <_io.TextIOWrapper name='/home/seb/showcase/roomba/sniffs/2024-07-01/cap0004-0310/stderr_59153b53-c49d-44de-99d2-b5a3490df29a.log' mode='w' encoding='UTF-8'>.
+
+ Capture complete. Saved to roomba_59153b53-c49d-44de-99d2-b5a3490df29a.pcap
+ tcpdump took 1.11 seconds.
+ Ensuring correct ownership of created files.
+ Saving metadata.
+ END  SNIFF SUBCOMMAND
+        
+    \end{minted}
+    \caption{Output with max verbosity and debug flag set.}
+    \label{fig:example-debug-output}
+\end{figure}

thesis/Back/AppendixB.tex

@@ -0,0 +1,16 @@
+\chapter{Appendix B}
+\section{Software Requirements}\label{sec:software-req}
+\iottbsc was developed on the \textit{Linux}\footnote{\url{kernel.org}} operating system \textit{Fedora 40}\footnote{\url{https://fedoraproject.org/workstation/}}. It has not been tested on any other platform.
+\iottbsc is implemented in a Python\footnote{\url{python.org}} package \iottb, which has been developed with Python version 3.12.
+
+\subsection{Runtime Dependencies}
+\begin{itemize}
+    \item Poetry\footnote{\url{https://python-poetry.org/}}, version 1.8.3. Used for packaging and dependency management.
+    \item Click\footnote{\url{https://click.palletsprojects.com/en/8.1.x/}}, version 8.1, is a library which enables parameter handling through decorated functions.
+\end{itemize}
+
+\subsection{Testing Dependencies}
+\begin{itemize}
+    \item Pytest\footnote{\url{https://docs.pytest.org/en/8.2.x/}}, versions 8.2. Although not many exist.
+    
+\end{itemize}

thesis/Back/CommandRef.tex

@@ -0,0 +1,145 @@
+\chapter{Appendix D}\label{appendix:cmdref}
+
+\section{\iottb}\label{cmdref:iottb}
+\begin{verbatim}
+Usage: iottb [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  -v, --verbosity  Set verbosity  [default: 0; 0<=x<=3]
+  -d, --debug      Enable debug mode
+  --dry-run        [default: True]
+  --cfg-file PATH  Path to iottb config file  [default:
+                   $HOME/.config/iottb/iottb.cfg]
+  --help           Show this message and exit.
+
+Commands:
+  add-device           Add a device to a database
+  init-db
+  rm-cfg               Removes the cfg file from the filesystem.
+  rm-dbs               Removes ALL(!) databases from the filesystem if...
+  set-key-in-table-to  Edit config or metadata files.
+  show-all             Show everything: configuration, databases, and...
+  show-cfg             Show the current configuration context
+  sniff                Sniff packets with tcpdump
+\end{verbatim}
+
+\subsection{Initialize Database}\label{cmdref:init-db}
+\begin{verbatim}
+Usage: iottb init-db [OPTIONS]
+
+Options:
+  -d, --dest PATH                 Location to put (new) iottb database
+  -n, --name TEXT                 Name of new database.  [default: iottb.db]
+  --update-default / --no-update-default
+                                  If new db should be set as the new default
+                                  [default: update-default]
+  --help                          Show this message and exit.
+\end{verbatim}
+
+\subsection{Add device}\label{cmdref:add-device}
+\begin{verbatim}
+Usage: iottb add-device [OPTIONS]
+
+  Add a device to a database
+
+Options:
+  --dev, --device-name TEXT   The name of the device to be added. If this
+                              string contains spaces or other special
+                              characters               normalization is
+                              performed to derive a canonical name  [required]
+  --db, --database DIRECTORY  Database in which to add this device. If not
+                              specified use default from config.  [env var:
+                              IOTTB_DB]
+  --guided                    Add device interactively  [env var:
+                              IOTTB_GUIDED_ADD]
+  --help                      Show this message and exit.
+\end{verbatim}
+
+\subsection{Capture traffic with \textit{tcpdump}}\label{cmdref:sniff}
+\begin{verbatim}
+Usage: iottb sniff [OPTIONS] [TCPDUMP-ARGS] [DEVICE]
+
+  Sniff packets with tcpdump
+
+Options:
+  Testbed sources: 
+    --db, --database TEXT  Database of device. Only needed if not current
+                           default.  [env var: IOTTB_DB]
+    --app TEXT             Companion app being used during capture
+  Runtime behaviour: 
+    --unsafe               Disable checks for otherwise required options.
+                           [env var: IOTTB_UNSAFE]
+    --guided               [env var: IOTTB_GUIDED]
+    --pre TEXT             Script to be executed before main  command is
+                           started.
+    --post TEXT            Script to be executed upon completion of main
+                           command.
+  Tcpdump options: 
+    -i, --interface TEXT   Network interface to capture on.If not specified
+                           tcpdump tries to find and appropriate one.  
+                           [env var: IOTTB_CAPTURE_INTERFACE]
+    -a, --address TEXT     IP or MAC address to filter packets by.  
+                           [env var: IOTTB_CAPTURE_ADDRESS]
+    -I, --monitor-mode     Put interface into monitor mode.
+    --ff TEXT              tcpdump filter as string or file path.  
+                           [env var: IOTTB_CAPTURE_FILTER]
+    -#, --print-pacno      Print packet number at beginning of line. True by
+                           default.  [default: True]
+    -e, --print-ll         Print link layer headers. True by default.
+    -c, --count INTEGER    Number of packets to capture.  [default: 1000]
+  --help                   Show this message and exit.    
+\end{verbatim}
+
+\section{Utility commands}\label{cmdref:sec:utils}
+Utility Commands mostly for development and have not yet been integrated into the standard workflow.
+\subsection{Remove Configuration}\label{cmdref:rm-cfg}
+\begin{verbatim}
+Usage: iottb rm-cfg [OPTIONS]
+
+  Removes the cfg file from the filesystem.
+
+  This is mostly a utility during development. Once non-standard database
+  locations are implemented, deleting this would lead to iottb not being able
+  to find them anymore.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.   
+\end{verbatim}
+
+\subsection{Remove Database}\label{cmdref:rm-dbs}
+\begin{verbatim}
+Usage: iottb rm-dbs [OPTIONS]
+
+  Removes ALL(!) databases from the filesystem if they're empty.
+
+  Development utility currently unfit for use.
+
+Options:
+  --yes   Confirm the action without prompting.
+  --help  Show this message and exit.    
+\end{verbatim}
+
+\subsection{Display Configuration File}\label{cmdref:show-cfg}
+\begin{verbatim}
+Usage: iottb show-cfg [OPTIONS]
+
+  Show the current configuration context
+
+Options:
+  --cfg-file PATH  Path to the config file  [default:
+                   /home/seb/.config/iottb/iottb.cfg]
+  -pp              Pretty Print
+  --help           Show this message and exit   
+\end{verbatim}
+
+\subsection{"Show All"}\label{cmdref:show-all}
+\begin{verbatim}
+Usage: iottb show-all [OPTIONS]
+
+  Show everything: configuration, databases, and device metadata
+
+Options:
+  --help  Show this message and exit.    
+\end{verbatim}
+

thesis/Chapters/ch1-introduction.tex

@@ -0,0 +1,44 @@
+% !TEX root = ../Thesis.tex
+\chapter{Introduction}\label{introduction}
+\iot devices are becoming increasingly prevalent in modern homes, offering a range of benefits such as controlling home lighting, remote video monitoring, and automated cleaning \citep{iothome2019}. 
+These conveniences are made possible by the sensors and networked communication capabilities embedded within these devices. 
+However, these features also pose significant privacy and security risks \citep{islamiot2023}. 
+IoT devices are often integrated into home networks and communicate over the internet with external servers, potentially enabling surveillance or unauthorized data sharing without the user's knowledge or consent \citep{infoexpiot}. Moreover, even in the absence of malicious intent by the manufacturer, these devices are still vulnerable to programming bugs and other security failures \citep{peekaboo2020}.
+\medskip
+
+Security researchers focused on the security and privacy of such \iot devices rely on various utilities and tools for conducting research. 
+These tools are often glued together in scripts with arbitrary decisions about file naming and data structuring. 
+Such impromptu scripts typically have a narrow range of application, making them difficult to reuse across different projects. Consequently, useful parts are manually extracted and incorporated into new scripts for each project, exacerbating the problem.
+\medskip
+
+This approach leads to scattered data, highly tailored scripts, and a lack of standardized methods for sharing or reproducing experiments. The absence of standardized tools and practices results in inconsistencies in data collection and storage, making it difficult to maintain compatibility across projects. 
+Furthermore, the lack of conventions about file naming and data structuring leads to issues in finding and accessing the data. 
+For research groups, these issues are further compounded during the onboarding of new members, who must navigate this fragmented landscape and often create their own ad-hoc solutions, perpetuating the cycle of inefficiency and inconsistency.
+\medskip
+
+To systematically and reproducibly study the privacy and security of IoT devices, an easy-to-use testbed that automates and standardizes various aspects of experimenting with IoT devices is needed.
+
+\section{Motivation}\label{sec:motivation}
+The primary motivation behind this project is to address the challenges faced by security researchers in the field of IoT device security and privacy. 
+The scattered nature of data, the lack of standardized tools, and the ad-hoc methods used for data collection or processing, are an obstacle for researchers who want to produce valid and reproducible results \citep{fursinckorg2021}. 
+A standardized testbed, enabling a more systematic approach to collecting and analyzing network data from \iot devices, can help make tedious and error-prone aspects of conducting experiments on \iot devices more bearable, while at the same time enhancing the quality of the data, by adhering to interoperability standards by establishing data collection and storage standards.
+This bachelor project is specifically informed by the needs of the PET research group at the University of Basel, who will utilize it to run IoT device experiments, and as a foundation to build more extensive tooling.
+
+\section{Goal}\label{sec:goal}
+The goal of this project is to design and implement a testbed for IoT device experiments. To aid reproducibility, there are two main objectives:
+
+First, the testbed should automate key aspects of running experiments with IoT devices, particularly the setup and initialization of data collection processes as well as some basic post-collection data processing.
+
+Secondly, the testbed should standardize how data from experiments is stored. This includes standardizing data and metadata organization, establishing a naming scheme, and defining necessary data formats.
+A more detailed description to how this is adapted for this project follows in \cref{ch:adaptation}.
+
+
+\section{Outline}
+This report documents the design and implementation of an \iot testbed.
+In the remainder of the text, the typographically formatted string "\iottbsc" refers to this projects' conception of testbed, whereas "\iottb" specifically denotes the Python package which is the implementation artifact from this project.
+
+This report outlines the general goals of a testbed, details the specific functionalities of \iottbsc, and explains how the principles of automation and standardization are implemented. 
+We begin by giving some background on the most immediately useful concepts.
+\cref{ch:adaptation} derives requirements for \iottbsc starting from first principles and concludes by delineating the scope considered for implementation, which is described in \cref{ch4}.
+In \cref{ch:5-eval} we evaluate \iottbsc, and more specifically, the \iottb software package against the requirements stated in \cref{ch:adaptation}.
+We conclude in \cref{ch:conclusion} with an outlook on further development for \iottbsc.

thesis/Chapters/ch2-background.tex

@@ -0,0 +1,48 @@
+% !TEX root = ../Thesis.tex
+\chapter{Background}
+This section provides the necessary background to understand the foundational concepts related to IoT devices, testbeds, and data principles that inform the design and implementation of \iottbsc.
+
+\section{Internet of Things}
+The \iot refers to the connection of “things” other than traditional computers to the internet. The decreasing size of microprocessors has enabled their integration into smaller and smaller objects. Today, objects like security cameras, home lighting, or children's toys may contain a processor and embedded software that enables them to interact with the internet. The Internet of Things encompasses objects whose purpose has a physical dimension, such as using sensors to measure the physical world or functioning as simple controllers. When these devices can connect to the internet, they are considered part of the Internet of Things and are referred to as \textbf{IoT devices} (see \citet{whatissmartdevice2018} and \citet{iotfundamentals}).
+
+\section{Testbed}
+A testbed is a controlled environment set up to perform experiments and tests on new technologies. The concept is used across various fields such as aviation, science, and industry. Despite the varying contexts, all testbeds share the common goal of providing a stable, controlled environment to evaluate the performance and characteristics of the object of interest.
+
+Examples of testbeds include:
+\begin{enumerate}
+    \item \textbf{Industry and Engineering}: In industry and engineering, the term \emph{platform} is often used to describe a starting point for product development. A platform in this context can be considered a testbed where various components and technologies are integrated and tested together before final deployment.
+    \item \textbf{Natural Sciences}: In the natural sciences, laboratories serve as testbeds by providing controlled environments for scientific experiments. For example, climate chambers are used to study the effects of different environmental conditions on biological samples (e.g., in \citet{vaughan2005use}). Another example is the use of wind tunnels in aerodynamics research to simulate and study the effects of airflow over models of aircraft or other structures.
+    \item \textbf{Computing}: In computing, specifically within software testing, a suite of unit tests, integrated development environments (IDEs), and other tools could be considered as a testbed. This setup helps in identifying and resolving potential issues before deployment. By controlling parameters of the environment, a testbed can ensure that the software behaves as expected under specified conditions, which is essential for reliable and consistent testing.
+    \item \textbf{Interdisciplinary}: Testbeds can take on considerable scales. For instance, in \citet{tbsmartgrid2013} provides insight into the aspects of  a testbed for a smart electric grid. 
+    This testbed is composed out of multiple systems, — an electrical grid, internet, and communication provision — which in their own right are already complex environments.
+    The testbed must, via simulation or prototyping, provide control mechanisms, communication, and physical system components.
+    
+\end{enumerate}  
+
+
+\section{FAIR Data Principles}
+\label{concept:fair}
+The \emph{FAIR Data Principles} were first introduced by \citet{wilkinson_fair_2016} with the intention to improve the reusability of scientific data. The principles address \textbf{F}indability, \textbf{A}ccessibility, \textbf{I}nteroperability, and \textbf{R}eusability. Data storage designers may use these principles as a guide when designing data storage systems intended to hold data for easy reuse. 
+For a more detailed description, see \citep{go-fair}.
+
+\section{Network Traffic}\label{sec:network-traffic}
+Studying \iot devices fundamentally involves understanding their network traffic behavior. 
+This is because network traffic contains (either explicitly or implicitly embedded in it) essential information of interest. 
+Here are key reasons why network traffic is essential in the context of \iot device security:
+\begin{enumerate}
+    \item \textbf{Communication Patterns}: Network traffic captures the communication patterns between IoT devices and external servers or other devices within the network. By analyzing these patterns, researchers can understand how data flows in and out of the device, which is critical for evaluating performance and identifying any unauthorized communications or unintended leaking of sensitive information.
+    \item \textbf{Protocol Analysis:} Examining the protocols used by IoT devices helps in understanding how they operate. Different devices might use various communication protocols, and analyzing these can reveal insights into their compatibility, efficiency, and security. Protocol analysis can also uncover potential misconfigurations or deviations from expected behavior.
+    \item \textbf{Flow Monitoring:} Network traffic analysis is a cornerstone of security research. It allows researchers to identify potential security threats such as data breaches, unauthorized access, and malware infections. By monitoring traffic, one can detect anomalies that may indicate security incidents or vulnerabilities within the device.
+    \item \textbf{Information Leakage}: \iot devices are often deployed in a home environment and connect to the network through wireless technologies \citep{iothome2019}. This allows an adversary to passively observe traffic. While often this traffic is encrypted, the network flow can leak sensitive information, which is extracted through more complex analysis of communication traffic and Wi-Fi packets \citep{friesssniffing2018}, \citep{infoexpiot}. In some cases, the adversary can determine the state of the smart environment and their users \citep{peekaboo2020}.
+\end{enumerate}
+
+
+\section{(Network) Packet Capture}
+Network \textit{packet capture} \footnote{also known as \emph{packet sniffing}, \emph{network traffic capture}, or just \emph{sniffing}. The latter is often used when referring to nefarious practices.} fundamentally describes the act or process of intercepting and storing data packets traversing a network. It is the principal technique used for studying the behavior and communication patterns of devices on a network. For the reasons mentioned in \cref{sec:network-traffic}, packet capturing is the main data collection mechanism used in \iot device security research, and also the one considered for this project.
+
+\section{Automation Recipes}
+\todoRevise()
+Automation recipes can be understood as a way of defining a sequence of steps needed for a process.
+In the field of machine learning, \textit{Collective Mind}\footnote{\url{https://github.com/mlcommons/ck}} provides a small framework to define reusable recipes for building, running, benchmarking and optimizing  machine learning applications.
+A key aspect of these recipes some platform-independent, which has enabled wider testing and benchmarking of machine learning models. Even if a given recipe is not yet platform independent, it can be supplemented with user-specific scripts which handle the platform specifics. Furthermore, it is possible to create a new recipe from the old recipe and the new script, which, when made accessible, essentially has extended the applicability of the recipe \citet{friesssniffing2018}.
+Automation recipes express the fact that some workflow is automated irrespective of the underlying tooling. A simple script or application can be considered an recipe (or part of) 

thesis/Chapters/ch3-adaptation.tex

@@ -0,0 +1,141 @@
+\chapter{Adaptation}\label{ch:adaptation}
+
+In this chapter, we outline the considerations made during the development of the IoT testbed, \iottbsc. 
+Starting from first principles, we derive the requirements for our testbed and finally establish the scope for \iottbsc.
+The implemented testbed which results from this analysis, the software package \iottb, is discussed in \cref{ch4}.\\
+
+\section{Principal Objectives}\label{sec:principles-and-objectives}
+The stated goal for this bachelor project (see \cref{sec:goal}), is to create a testbed for \iot devices, which automates aspects of the involved workflow,  with the aim of increasing reproducibility, standardization, and compatibility of tools and data across project boundaries.
+We specify two key objectives supporting this goal:
+\begin{enumerate}[label=\textit{Objective \arabic*}]
+    \item \textbf{Automation Recipes:}\label{obj:recipies} The testbed should support specification and repeated execution of important aspects of experiments with IoT devices, such as data collection and analysis (see \citep{fursinckorg2021})
+    \item \textbf{FAIR Data Storage:}\label{obj:fair} The testbed should store data in accordance with the FAIR \citep{go-fair} principles.
+\end{enumerate}
+
+\section{Requirements Analysis}\label{sec:requirements}
+In this section, we present the results of the requirements analysis based on the principal objectives.
+The requirements derived for \ref{obj:recipies} are presented in \cref{table:auto_recipe_requirements}.
+\cref{table:fair_data_storage_requirements} we present requirements based on \ref{obj:fair}.
+
+\begin{table}[H]
+\centering
+\caption{Automation Recipes Requirements}
+\label{table:auto_recipe_requirements}
+\begin{minipage}{\textwidth}
+\begin{enumerate}[label=\textit{R1.\arabic*}]
+    \item \label{req:auto_install_tools} \textbf{Installation of Tools}: Support installation of necessary tools like \textit{mitmproxy} \cite{mitmproxy}, \textit{Wireshark} \cite{wiresharkorg} or \textit{tcpdump} \cite{tcpdump}).
+    
+        \textit{Reasoning:} 
+        There are various tools used for data collection and specifically packet capture.
+        Automating the installation of necessary tools ensures that all required software is available and configured correctly without manual intervention. This reduces the risk of human error during setup and guarantees that the testbed environment is consistently prepared for use. Many platforms, notably most common Linux distributions, come with package managers which provide a simple command-line interface for installing software while automatically handling dependencies. This allows tools to be quickly installed, making it a \textit{lower priority} requirement for \iottbsc.
+
+    \item \label{req:auto_config_start} \textbf{Configuration and Start of Data Collection}: Automate the configuration and start of data collection processes. Specific subtasks include:
+    \begin{enumerate}
+        \item Automate wireless hotspot management on capture device.
+        \item Automatic handling of network capture, including the collection of relevant metadata.
+    \end{enumerate}
+    
+        \textit{Reasoning:} 
+        Data collection is a central step in the experimentation workflow. Configuration is time-consuming and prone to error, suggesting automating this process is useful.As mentioned in \cref{sec:motivation}, current practices lead to incompatible data and difficult to reuse scripts.
+        Automating the configuration and start of data collection processes ensures a standardized approach, reducing the potential for user error 
+        and thereby increasing data compatibility and efficient use of tools. Automating this process must be a central aspect of \iottbsc.
+
+    \item \label{req:auto_data_processing} \textbf{Data Processing}: Automate data processing tasks.
+    
+        \textit{Reasoning:} Some network capture tools produce output in a binary format. To make the data available to other processes, often the data must be transformed in some way.
+        Data processing automation ensures that the collected data is processed uniformly and efficiently, enhancing it reusability and interoperability. Processing steps may include cleaning, transforming, and analyzing the data, which are essential steps to derive meaningful insights. Automated data processing saves time and reduces the potential for human error. It ensures that data handling procedures are consistent, which is crucial for comparing results across different experiments and ensuring the validity of findings.
+    
+    
+    \item \label{req:auto_reproducibility} \textbf{Reproducibility}: Ensure that experiments can be repeated with the same setup and configuration.
+
+        \textit{Reasoning:} A precondition to reproducible scientific results is the ability to run experiments repeatedly with all relevant aspects are set up and configured identically.
+    \item \label{req:auto_execution_control} \textbf{Execution Control}: Provide mechanisms for controlling the execution of automation recipes (e.g., start, stop, status checks).
+    
+        \textit{Reasoning:} Control mechanisms are essential for managing the execution of automated tasks. This includes starting, stopping, and monitoring the status of these tasks to ensure they are completed successfully.
+
+    \item \label{req:auto_error_logging} \textbf{Error Handling and Logging}: Include robust error handling and logging to facilitate debugging to enhance reusability.
+
+        \textit{Reasoning:} Effective error handling and logging improve the robustness and reliability of the testbed.Automation recipes may contain software with incompatible logging mechanisms. 
+        To facilitate development and troubleshooting, a unified and principled logging important for \iottbsc.
+    \item \label{req:auto_documentation} \textbf{Documentation}: Provide clear documentation and examples for creating and running automation recipes.
+\end{enumerate}
+\end{minipage}
+\end{table}
+
+\begin{table}[H]
+\centering
+\caption{FAIR Data Storage Requirements}
+\label{table:fair_data_storage_requirements}
+\begin{minipage}{\textwidth}
+\begin{enumerate}[label=\textit{R2.\arabic*}]
+    \item \label{req:fair_data_meta_inventory} \textbf{Data and Metadata Inventory}: \iottbsc should provide an inventory of data and metadata that typically need to be recorded (e.g., raw traffic, timestamps, device identifiers).
+    
+    \textit{Reasoning:} Providing a comprehensive inventory of data and metadata ensures that data remains findable after collection. Including metadata increases interpretability and gives context necessary for extracting reproducible results.
+    
+    \item \label{req:fair_data_formats} \textbf{Data Formats and Schemas}: Define standardized data formats and schemas.
+    
+        \textit{Reasoning:} Standardized data formats and schemas ensure consistency and interoperability.
+    
+    \item \label{req:fair_file_naming} \textbf{File Naming and Directory Hierarchy}: Establish clear file naming conventions and directory hierarchies. for organized data storage. 
+    
+        \textit{Reasoning:} This enhances findability and accessibility.
+    \item \label{req:fair_preservation} \textbf{Data Preservation Practices}: Implement best practices for data preservation, including recommendations from authoritative sources like the Library of Congress \citep{recommendedformatrsLOC}.
+    
+        \textit{Reasoning:} Implementing best practices for data preservation can mitigate data degradation and ensures integrity of data over time. This ensures long-term accessibility and reusability.
+    \item \label{req:fair_accessibility} \textbf{Accessibility Controls}: Ensure data accessibility with appropriate    permissions and access controls.
+    \item \label{req:fair_interoperability} \textbf{Interoperability Standards}: Use widely supported formats and protocols to facilitate data exchange and interoperability.
+    \item \label{req:fair_reusability} \textbf{Reusability Documentation}: Provide detailed metadata to support data reuse by other researchers.
+\end{enumerate}
+\end{minipage}
+\end{table}
+
+We return to these when we evaluate \iottbsc in \cref{ch:5-eval}.
+
+\section{Scope}\label{sec:scope}
+This section defines the scope of the testbed \iottbsc.
+To guide the implementation of the software component of this bachelor project,  \iottb,
+we focus on a specific set of requirements that align with the scope of a bachelor project. 
+While the identified requirements encompass a broad range of considerations, we have prioritized those that are most critical to achieving the primary objectives of the project.
+
+For this project, we delineate our scope regarding the principal objectives as follows:
+\begin{itemize}
+    \item \ref{obj:recipies}: \iottb focuses on complying with \ref{req:auto_config_start}, \ref{req:auto_reproducibility}.
+    \item \ref{obj:fair}: \iottb ensures FAIR data storage implicitly, with the main focus lying on \ref{req:fair_data_formats}, \ref{req:fair_data_meta_inventory}, \ref{req:fair_file_naming}.
+\end{itemize}
+
+
+\subsection{Model Environment}\label{sec:assumed-setup}
+In this section, we describe the environment model assumed as the basis for conduction \iot device experiments.
+This mainly involves delineating the network topology. Considerations are taken to make this environment, over which the \iottb testbed software has no control, easy reproducible \citep{vacuumpie2023}.\\
+
+We assume that the \iot device generally requires a Wi-Fi connection. 
+This implies that the environment is configured to reliably capture network traffic without disrupting the \iot device's connectivity. This involves setting up a machine with internet access (wired or wireless) and possibly one Wi-Fi card supporting AP mode to act as the \ap for the \iot device under test \citep{surveytestingmethods2022}. 
+Additionally, the setup must enable bridging the IoT-AP network to the internet to ensure  \iot device.\\
+
+Specifically, the assumed setup for network traffic capture includes the following components:
+\begin{enumerate}
+    \item \textbf{IoT Device:} The device under investigation, connected to a network.
+    \item \textbf{Capture Device:} A computer or dedicated hardware device configured to intercept and record network traffic. This is where \iottb runs. 
+    \item \textbf{Wi-Fi \ap:} The \ap through which the \iot device gets network access.
+    \item \textbf{Router/ Internet gateway:} The network must provide internet access.
+    \item \textbf{Switch or software bridge:} At least either a switch or an \os with software bridge support must be available to be able to implement one of the setups described in \cref{fig:cap-setup1} and \cref{fig:cap-setup2}.
+    \item \textbf{Software:} tcpdump is needed for network capture.
+\end{enumerate}
+\newpage
+\begin{figure}[!ht]
+    \centering
+    \includegraphics[width=0.75\linewidth]{Figures/network-setup1.png}
+    \caption{Capture setup with separate Capture Device and AP}
+    \label{fig:cap-setup1}
+\end{figure}
+
+\begin{figure}[!ht]
+    \centering
+    \includegraphics[width=0.75\linewidth]{Figures/setup2.png}
+    \caption{Capture setup where the capture device doubles as the \ap for the \iot device.}
+    \label{fig:cap-setup2}
+\end{figure}
+\newpage
+
+
+

thesis/Chapters/ch4-iottb.tex

@@ -0,0 +1,153 @@
+\chapter{Implementation}\label{ch4}
+This chapter discusses the implementation of the IoT device testbed, \iottbsc which is developed using the Python programming language. This choice is motivated by Python's wide availability and the familiarity many users have with it, thus lowering the barrier for extending and modifying the testbed in the future. The testbed is delivered as a Python package and provides the \iottb command with various subcommands. A full command reference can be found at \cref{appendix:cmdref}.\\
+Conceptually, the software implements two separate aspects: data collection and data storage. 
+The \iottbsc database schema is implicitly implemented by \iottb. Users use \iottb mainly to operate on the database or initiate data collection. Since the database schema is transparent to the user during operation, we begin with a brief description of the database layout as a directory hierarchy, before we get into \iottb \cli.
+
+\section{Database Schema}
+The storage for \iottbsc is implemented on top of the file system of the user.
+Since user folder structures provide little standardization, we require a configuration file, while gives \iottb some basic information about the execution environment.
+The testbed is configured in a configuration file in JSON format, following the scheme in \cref{lst:cfg-shema}.
+\verb|DefaultDatabase| is a string which represents the name of the database, which is a directory in \\
+\verb|DefaultDatabasePath| once initialized.
+\iottb assumes these values during execution, unless the user specified otherwise. 
+If the user specifies a different database location as in option in a subcommand, \verb|DatabaseLocations| is consulted.
+\verb|DatabaseLocations| is a mapping from every known database name to the full path of its parent directory in the file system.
+The configuration file is loaded for every invocation of \iottb.
+It provides the minimal operating information.
+Now that we understand
+\begin{listing}[!ht]
+    \inputminted[]{json}{cfg-shema.json}
+    \caption{Schema of the testbed configuration file.}
+    \label{lst:cfg-shema}
+\end{listing}
+\newpage
+\section{High Level Description}
+\iottb is invoked following the schema below. In all cases, a subcommand be specified for anything to happen.
+\iottb is used from the command line and follows the following schema:
+\begin{minted}[fontsize=\small]{bash}
+iottb [<global options>] <subcommand> [<subcommand options>] [<argument(s)>]
+\end{minted}
+\todoRevise{Better listing}
+When \iottb is invoked, it first checks to see if it can find the database directory in the \os users home directory\footnote{Default can be changed}. 
+
+\section{Database Initialization}\label{sec:db-init}
+The IoT testbed database is defined to be a directory named \db. Currently, \iottb creates this directory in the user's home directory (commonly located at the path \texttt{/home/<username>} on Linux systems) the first time any subcommand is used. All data and metadata are placed under this directory. Invoking \verb|iottb init-db| without arguments causes defaults to be loaded from the configuration file. If the file does not exist, it is created with default values following \cref{lst:cfg-shema}. Else, the database is created with the default name or the user-suplied name as a directory in the file system, unless a database under that name is already registered in the \verb|DatabaseLocaions| map. The commands described in the later sections all depend on the existence of a \iottbsc database.
+It is neither possible to add a device nor initiate data collection without an existing database.
+The full command line specification can be found in \cref{cmdref:init-db}.
+Once a database is initialized, devices may be added to that database.
+
+\section{Adding Devices}\label{sec:add-dev}
+Before we capture the traffic of a \iot device, \iottb demands that there exists a dedicated
+directory for it. 
+We add a device to the database by passing a string representing the name of the device to the \addev subcommand.
+This does two things:
+\begin{enumerate}
+    \item A python object is initialized from the class as in \cref{lst:dev-meta-python}
+    \item A directory for the device is created as \verb|<db-path>/<device_canonical_name>|
+    \item A metadata file \verb|device_metadata.json| is created and placed in the newly created directory. This file is in             the JSON format, and follows the schema seen in \cref{lst:dev-meta-python}.
+\end{enumerate}
+
+\begin{listing}[!ht]
+    \inputminted[firstline=12, lastline=29, linenos]{python}{device_metadata.py}
+    \caption{Device Metadata}
+    \label{lst:dev-meta-python}
+\end{listing}
+
+The Device ID is automatically generated using a UUID to be FAIR compliant. \verb|canonical_name| is generated by the \verb|make_canonical_name()| function provided in \cref{lst:dev-canonical}. 
+Fields not supplied to \verb|__init__| in \cref{lst:dev-meta-python} are kept empty. The other fields in  are currently not used by \iottb itself, but provide metadata 
+which can be used during a processing step. Optionally, one can manually create such a file with pre-set values and pass it to the setup.
+For example, say the testbed contains a configuration as can be seen in \cref{lst:appendix:appendixa:config-file}
+
+\begin{listing}[!ht]
+    \inputminted[firstline=1, lastline=8, linenos]{json}{appendixa-after-add-device-dir.txt}
+    \caption{Directory layout after adding device 'default' and 'Roomba'}
+    \label{lst:cfg-file-post-add}
+\end{listing}
+
+If we then add two devices \verb|'iPhone 13 (year 2043)'| and \verb|roomba|, the layout of the database resembles \cref{lst:cfg-db-layout-post-add} and, for instance, the \verb|roomba| devices' will contain the metadata listed in \cref{lst:meta-roomba-post-add}. See \cref{appendixA:add-dev-cfg} for a complete overview.
+
+\begin{listing}[!ht]
+    \lstinputlisting[firstline=11, lastline=16]{appendixa-after-add-device-dir.txt}
+    \caption{Directory layout after adding device 'default' and 'Roomba'}
+    \label{lst:cfg-db-layout-post-add}
+\end{listing}
+
+\begin{listing}[!ht]
+    \lstinputlisting[firstline=39, lastline=55]{appendixa-after-add-device-dir.txt}
+    \caption{Directory layout after adding device 'default' and 'Roomba'}
+    \label{lst:meta-roomba-post-add}
+\end{listing}
+
+\newpage
+\section{Traffic Sniffing}\label{sec:sniff}
+Automated network capture is a key component of \iottb. The standard network capture is provided by the \texttt{sniff} subcommand, which wraps the common traffic capture utility \emph{tcpdump}\citep{tcpdump}. \cref{cmdref:sniff} shows usage of the command.
+
+Unless explicitly allowed by specifying that the command should run in \texttt{unsafe} mode, an IPv4, or MAC address \emph{must} be provided. An IP addresses are only accepted in dot-decimal notation \footnote{e.g., 172.168.1.1} and MAC addresses must specify as six groups of two hexadecimal digits\footnote{e.g., 12:34:56:78:AA:BB}. Failing to provide either results in the capture being aborted. The rationale behind this is simple: they are the only way to identify the traffic of interest. Of course, it is possible to retrieve the IP or MAC after a capture. Still, the merits outweigh the annoyance. The hope is that this makes \iottb easier to use \emph{correctly}. For example, consider the situation, where a student is tasked with performing multiple captures across multiple devices. If the student is not aware of the need of an address for the captured data to be usable, then this policy avoids the headache and frustration of wasted time and unusable data. 
+
+To comply with \ref{req:auto_config_start} and \ref{req:fair_data_meta_inventory}, each capture also stores some metadata in \texttt{capture\_metadata.json}. \cref{lst:cap-meta} shows the metadata files schema.
+
+
+\begin{listing}[!ht]
+\inputminted[firstline=288, lastline=319]{python}{sniff.py}
+\caption{Metadata Stored for sniff command}
+\label{lst:cap-meta}
+\end{listing}
+
+The \texttt{device\_id} is the \uuid \ of the device for which the capture was performed. This ensures the capture metadata remains associated even if files are moved. Furthermore, each capture also gets a \uuid. This \uuid \ is used as the suffix for the PCAP file, and the log files. The exact naming scheme is given in \cref{lst:cap-naming}.
+
+\begin{listing}
+\inputminted[firstline=179, lastline=181]{python}{sniff.py}
+\caption{Naming scheme for files created during capture.}
+\label{lst:cap-naming} 
+\end{listing}
+
+
+
+\section{Working with Metadata}
+The \texttt{meta} subcommand provides a facility for manipulating metadata files. It allows users to get the value of any key in a metadata file as well as introduce new key-value pairs. However, it is not possible to change the value of any key already present in the metadata. This restriction is in place to prevent metadata corruption.
+
+The most crucial value in any metadata file is the \texttt{uuid} of the device or capture the metadata belongs to. Changing the \texttt{uuid} would cause \iottb to mishandle the data, as all references to data associated with that \texttt{uuid} would become invalid. Changeing the any other value might not cause mishandling by \iottb, but they nonetheless represent essential information about the data. Therefore, \iottb does not allow changes to existing keys once they are set.
+
+Future improvements might relax this restriction by implementing stricter checks on which keys can be modified. This would involve defining a strict set of keys that are write-once and then read-only.
+
+\section{Raw Captures}
+The \texttt{raw} subcommand offers a flexible way to run virtually any command wrapped in \iottb. Of course, the intended use is with other capture tools, like \textit{mitmproxy}\citet{mitmproxy}, and not arbitrary shell commands. 
+While some benefits, particularly those related to standardized capture, are diminished, users still retain the advantages of the database.
+
+
+The syntax of the \texttt{raw} subcommand is as follows:
+\begin{minted}{bash}
+iottb raw <device> <command-name> "<command-options-string>" # or
+iottb raw <device> "<string-executable-by-a-shell>" #
+\end{minted}
+
+\iottb does not provide error checking for user-supplied arguments or strings. 
+Users benefit from the fact that captures will be registered in the database, assigned a \texttt{uuid}, and associated with the device. 
+The metadata file of the capture can then be edited manually if needed.
+
+
+\iottb does not provide error checking for user-supplied arguments or strings. 
+Users benefit from the fact that captures will be registered in the database, assigned a \texttt{uuid}, and associated with the device. 
+The metadata file of the capture can then be edited manually if needed.
+
+However, each incorrect or unintended invocation that adheres to the database syntax (i.e., the specified device exists) will create a new capture directory with a metadata file and \texttt{uuid}. Therefore, users are advised to thoroughly test commands beforehand to avoid creating unnecessary clutter.
+
+\section{Integrating user scripts}\label{sec:integrating-user-scripts}
+The \texttt{--pre} and \texttt{--post} options allow users to run any executable before and after any subcommand, respectively. 
+Both options take a string as their argument, which is passed as input to a shell and launched as a subprocess. 
+The rationale for running the process in a shell is that Python's Standard Library process management module, \texttt{subprocess}\footnote{\url{https://docs.python.org/3/library/subprocess.html}}, does not accepts argument to the target subprocess when a single string is passed for execution.
+
+Execution is synchronous: the subcommand does not begin execution until the \texttt{--pre} script finishes, and the \texttt{--post} script only starts executing after the subcommand has completed its execution. \iottb always runs in that order. 
+
+There may be cases where a script provides some type of relevant interaction intended to run in parallel with the capture. Currently, the recommended way to achieve this is to wrap the target executable in a script that forks a process to execute the target script, detaches from it, and returns.
+
+These options are a gateway for more complex environment setups and, in particular, allow users to reuse their scripts, thus lowering the barrier to adopting \iottb.
+
+\section{Extending and Modifying the Testbed}
+One of the key design goals of \iottb is easy extensibility. \iottb uses the Click Library \citep{click} to handle parsing arguments. Adding a new command amounts to no more than writing a function and decorating it according to Click specification.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%% Figures
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+

thesis/Chapters/ch5-evaluation.tex

@@ -0,0 +1,202 @@
+\chapter{Evaluation}\label{ch:5-eval}
+In this sectioned we evaluate \iottb, paying particular attention to the requirements defined in \cref{sec:requirements}.
+
+\begin{table}[h!]
+\centering
+\begin{tabular}{|c|l|c|}
+\hline
+\textbf{Requirement ID} & \textbf{Description} & \textbf{Status} \\ \hline
+\ref{req:auto_install_tools} & Installation of Tools & Not Met \\ \hline
+\ref{req:auto_config_start} & Configuration and Start of Data Collection & $\downarrow$ \\ \hline
+\ref{req:auto_config_start}a) & Automate Wi-Fi Setup & Not Met \\ \hline
+\ref{req:auto_config_start}b) & Automate Data Capture & Met \\ \hline
+\ref{req:auto_data_processing} & Data Processing & Partially Met \\ \hline
+\ref{req:auto_reproducibility} & Reproducibility & Partially Met \\ \hline
+\ref{req:auto_execution_control} & Execution Control & Not Met \\ \hline
+\ref{req:auto_error_logging} & Error Handling and Logging & Partially Met \\ \hline
+\ref{req:auto_documentation} & Documentation & $\downarrow$ \\ \hline
+\ref{req:auto_documentation}a) & User Manual & Met \\ \hline
+\ref{req:auto_documentation}b) & Developer Docs & Not Met \\ \hline
+\ref{req:fair_data_meta_inventory} & Data and Metadata Inventory & Met \\ \hline
+\ref{req:fair_data_formats} & Data Formats and Schemas & Met \\ \hline
+\ref{req:fair_file_naming} & File Naming and Directory Hierarchy & Met \\ \hline
+\ref{req:fair_preservation} & Data Preservation Practices & Partially Met \\ \hline
+\ref{req:fair_accessibility} & Accessibility Controls & Not Met \\ \hline
+\ref{req:fair_interoperability} & Interoperability Standards & Fully Met \\ \hline
+\ref{req:fair_reusability} & Reusability Documentation & Met \\ \hline
+\end{tabular}
+\caption{Summary of Requirements Evaluation}
+\label{tab:requirements-evaluation}
+\end{table}
+
+\cref{tab:requirements-evaluation} gives an overview of the requirements introduced in \cref{sec:requirements} and our assessment of their status. 
+It is important to note that the status “Met” does not imply that the requirement is implemented to the highest possible standard. 
+Furthermore, this set of requirements itself can (and should) be made more specific and expanded in both detail and scope as the project evolves.
+
+Additionally, \ref{tab:requirements-evaluation} does not provide granularity regarding the status of individual components, which might meet the requirements to varying degrees. 
+For example, while the requirement for data collection automation may be fully met in terms of basic functionality, advanced features such as handling edge cases or optimizing performance might still need improvement. 
+Similarly, the requirement for data storage might be met in terms of basic file organization but could benefit from enhanced data preservation practices.
+
+Thus, the statuses presented in \cref{tab:requirements-evaluation} should be viewed as a general assessment rather ground truth. 
+Future work should aim to refine these requirements and their implementation to ensure that \iottbsc continues to evolve and improve.
+
+To provide a more comprehensive understanding, the following sections offer a detailed evaluation of each requirement. This detailed analysis will discuss how each requirement was addressed, the degree to which it was met, and any specific aspects that may still need improvement. By examining each requirement individually, we can better understand the strengths and limitations of the current implementation and identify areas for future enhancement.
+
+\section{\ref{req:auto_install_tools}: Installation of Tools}
+\textbf{Status: Not Met} \\
+\iottbsc does not install any software or tools by itself. Dependency management for Python packages is handled by installers like PIP, since the Python package declares its dependencies.
+Tcpdump is the only external dependency, and \iottbsc checks if Tcpdump is available on the capture device. If it is not, the user is asked to install it.
+Our position is that generally it is a good idea to not force installation of software and allow users the freedom to choose. The added benefit to the user of a built-in installer seems low. Adding some installer to \iottbsc does not promise great enough improvement in ease-of-use vis-à-vis the higher maintenance cost introduced to maintain such a module.
+For future work, this requirement could be removed.
+
+\section{\ref{req:auto_config_start}: Configuration and Start of Data Collection}
+\textbf{Status: Partially Met} \\
+The testbed automates the configuration and initiation of data collection processes, including wireless hotspot management and network capture. This automation reduces setup time and minimizes errors.
+The testbed automates some aspects of configuring and initializing the data collection process. This project focused on package capture and adjacent tasks. \ref{req:auto_config_start}b can be considered \textit{complete} in that packet capture is fully supported thorough Tcpdump and important metadata is saved. Depending on the setup (see \cref{fig:cap-setup1} and \cref{fig:cap-setup2}) a Wi-Fi hotspot needs to be set up before packet capture is initiated. \iottbsc does not currently implement automated setup and takedown of a hotspot on any platform, so \ref{req:auto_config_start} a is not currently met. There are scripts for Linux systems bundled with the Python package which can be used with the \texttt{--pre} and \texttt{--post} options mentioned in \cref{sec:integrating-user-scripts}. But to consider this task fully automated and supported, this should be built in to \iottbsc itself.
+Furthermore, there are other data collection tools like \textit{mitmproxy}\citep{mitmproxy} or more complicated setup tasks like setting up a routing table to allow for more capture scenarios, which are tedious tasks and lend themselves to automation. Future work should include extending the set of available automation recipes continuously. 
+New task groups/recipe domains should be added as sub-requirements of \ref{req:auto_config_start}.
+We propose the following new sub-requirement
+\begin{itemize}
+    \item \ref{req:auto_config_start}c: Testbed should implement automatic setup of NAT routing for situations where \ap is connection to the capture device and a bridged setup is not supported.
+    \item \ref{req:auto_config_start}d: Testbed should dynamically determine which type of hotspot setup is possible and choose the appropriate automation recipe.
+\end{itemize}
+Extending \ref{req:auto_config_start} means stating which data collection and adjacent recipes are wanted.
+
+\section{\ref{req:auto_data_processing}: Data Processing}
+\textbf{Status: Partially Met} \\
+While the testbed includes some basic data processing capabilities, there is room for improvement. 
+Currently, the only one recipe exists for processing raw data. \iottbsc can extract a CSV file from a PCAP file. The possibilities for automation recipes which support data processing are many.
+Having the data in a more standardized format allows for the creation of more sophisticated feature extraction recipes with application for machine learning. 
+Before they are available, users can still use the \texttt{--post} option with their feature extraction scripts.
+
+\section{\ref{req:auto_reproducibility}: Reproducibility}
+\textbf{Status: Met} \\
+Supported automation can be run with repeatedly, and used options are documented in the capture metadata. This allows others to repeat the process with the same options. 
+So in this respect, this requirement is met. But, the current state can be significantly improved by automating the process of repeating a capture task with the same configuration as previous captures.
+To support this, we propose the following new subrequirement which aids the automated reproduction of past capture workflows
+\begin{itemize}
+    \item \ref{req:auto_reproducibility}a: The testbed should be able to read command options from a file
+    \item \cref{req:auto_reproducibility}b: The testbed should be able to perform a capture based on metadata files of completed captures.
+\end{itemize}
+Taking these requirements promises to seriously increase reproducibility.
+
+\section{\ref{req:auto_execution_control}: Execution Control}
+\textbf{Status: Not Met} \\
+The testbed currently provides no controlled method to interfere with a running recipe. In most cases, \iottb will gracefully end if the user sends the process a SIGINT, but there are no explicit protections against data corruption in this case. Furthermore, during execution, \iottb writes to log files and prints basic information to the users' terminal. Extending this with a type of monitoring mechanism would be good steps toward complying with this requirement in the future.
+
+\section{R1.6: Error Handling and Logging}
+\textbf{Status: Met} \\
+Robust error handling and logging are implemented, ensuring that issues can be diagnosed and resolved effectively. Detailed logs help maintain the integrity of experiments. It is also possible for the user to control how much output is given in the terminal. Here are four examples of the same command, with just increasing degrees of verbosity specified by the user:
+
+\subsection{Logging Example}
+\textbf{Command: } \verb|iottb sniff roomba  --unsafe -c 10 <verbosity>|
+Verbosity can be unspecified, \verb|-v|, \verb|-vv| or \verb|-vvv|
+
+\begin{figure}
+    \centering
+\begin{minted}[breaklines]{bash}
+$ iottb sniff roomba  --unsafe -c 10
+Testbed [I]
+ Using canonical device name roomba
+ Found device at path /home/seb/showcase/roomba
+ Using filter None
+ Files will be placed in /home/seb/showcase/roomba/sniffs/2024-07-01/cap0000-0214
+ Capture has id 62de82ad-3aa2-460e-acd0-546e46377987
+ Capture setup complete!
+ Capture complete. Saved to roomba_62de82ad-3aa2-460e-acd0-546e46377987.pcap
+ tcpdump took 2.16 seconds.
+ Ensuring correct ownership of created files.
+ Saving metadata.
+ END  SNIFF SUBCOMMAND    
+\end{minted}
+    \caption{No verbosity.}
+    \label{fig:example-no-verb}
+\end{figure}
+
+On the first verbosity level, only logger warnings are now printed to the standard output. During normal execution we do not expect significantly more output. This is also true for the second verbosity level.
+\begin{figure}
+    \centering
+    \begin{minted}{bash}
+$ iottb -v|-vv sniff roomba  --unsafe -c 10
+<_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'>
+WARNING - iottb_config - DatabaseLocations are DatabaseLocationMap in the class iottb.models.iottb_config
+\end{minted}
+    \caption{Only \textit{additional} output for \-v or \-vv.}
+    \label{fig:example-one-verb}
+\end{figure}
+
+This changes once we reach the third verbosity level, because now additonally the logger level is set to "INFO".
+Clearly, \cref{fig:example-lvl-three} contains far more output than \cref{fig:example-one-verb}.
+It is possible to get even more output printed to standard output by also passing the \verb|--debug| flag.
+This produces significantly more output as can be seen in \cref{fig:example-debug-output}.
+\begin{figure}
+    \centering
+    \begin{minted}{bash}
+$ iottb -vvv sniff roomba  --unsafe -c 10
+<_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'>
+INFO - main - cli - 48 - Starting execution.
+INFO - iottb_config - __init__ - 24 - Initializing Config object
+WARNING - iottb_config - warn - 21 - DatabaseLocations are DatabaseLocationMap in the class iottb.models.iottb_config
+INFO - iottb_config - load_config - 57 - Loading configuration file
+INFO - iottb_config - load_config - 62 - Config file exists, opening.
+INFO - sniff - validate_sniff - 37 - Validating sniff...
+INFO - sniff - sniff - 91 - sniff command invoked
+INFO - string_processing - make_canonical_name - 20 - Normalizing name roomba
+Testbed [I]
+ Using canonical device name roomba
+ Found device at path /home/seb/showcase/roomba
+INFO - sniff - sniff - 152 - Generic filter None
+ Using filter None
+ Files will be placed in /home/seb/showcase/roomba/sniffs/2024-07-01/cap0003-0309
+ Capture has id f1e92062-4a82-4429-996c-97bd7fa57bec
+INFO - sniff - sniff - 186 - pcap file name is roomba_f1e92062-4a82-4429-996c-97bd7fa57bec.pcap
+INFO - sniff - sniff - 187 - stdout log file is stdout_f1e92062-4a82-4429-996c-97bd7fa57bec.log
+INFO - sniff - sniff - 188 - stderr log file is stderr_f1e92062-4a82-4429-996c-97bd7fa57bec.log
+INFO - sniff - sniff - 246 - tcpdump command: sudo tcpdump -# -n -vvv -c 10 -w /home/seb/showcase/roomba/sniffs/2024-07-01/cap0003-0309/roomba_f1e92062-4a82-4429-996c-97bd7fa57bec.pcap
+ Capture setup complete!
+ Capture complete. Saved to roomba_f1e92062-4a82-4429-996c-97bd7fa57bec.pcap
+ tcpdump took 2.12 seconds.
+ Ensuring correct ownership of created files.
+ Saving metadata.
+ END  SNIFF SUBCOMMAND        
+    \end{minted}
+    \caption{Caption}
+    \label{fig:example-lvl-three}
+\end{figure}
+
+\section{\ref{req:auto_documentation}: Documentation}
+\textbf{Status: Partially Met} \\
+For users, there is a 'Command Line Reference' (see \cref{appendix:cmdref}) which details all important aspects of operating the \iottb \cli. Furthermore, helpful messages are displayed regarding the correct syntax of the commands if an input is malformed. So user documentation does exist and, while certainly can be improved upon, is already helpful. 
+Unfortunately, documentation for developers is currently poor. The codebase is not systematically documented and there is currently no developer's manual. 
+Thoroughly documenting the existing codebase should be considered the most pressing issue and tackled first to improve developer documentation.
+
+\section{\ref{req:fair_data_meta_inventory}: Data and Metadata Inventory}
+\textbf{Status: Fully Met} \\
+The testbed organizes data and metadata in a standardized and principled way. The database is complete with respects to the currently primary and secondary artifact which stem from operating  \iottb itself.
+While complete now, extending \iottb carries the risk of breaking this requirement if not careful attention is given.
+Since the database is a central part of the system as a whole, extensions must ensure that they comply with this requirement before they get built in. 
+
+\section{\ref{req:fair_data_formats}: Data Formats and Schemas}
+\textbf{Status:  Met} \\
+The testbed standardizes directory and file naming. All metadata is in plain test and in the JSON format. This makes them very accessible to both humans and machines. Currently, the only binary format which \iottbsc creates are PCAP files. Luckily, the PCAP format is widely known and not proprietary, and  widely available tools (e.g., Wireshark\citep{wiresharkorg}) exist to inspect them. Furthermore, the data in the PCAP files can be extracted in to the plaintext CSV format, this further improves interoperability. Consistence is currently implicitly handles, that is, there are no strict schemas \footnote{Strict schemas for metadata file briefly were introduced, but then abandoned due to the lack of knowledge surrounding the PYdantic library \citep{pydantic}.} \iottb should generally not corrupt data during operation. But plaintext files are manually editable and can inadvertently be corrupted or made invalid (e.g. accidentally deleting a few digits from a UUID). 
+It is important to keep this in mind when extending \iottbsc and the types of files residing in the database become more heterogeneous.
+
+
+\subsection{\ref{req:fair_file_naming}: File Naming and Directory Hierarchy}
+\textbf{Status: Met} \\
+\iottb currently names all files which it creates according to a well-defined schema. In all cases, the file name is easily legible (e.g., metadata files like \cref{lst:cap-meta}) or the context of where the file resides provides easy orientation to a human reviewer. For instance, raw data files, which currently only are PCAP files, are all named with a \uuid. This is not helpful to the human, but the metadata file, which resides in the same directory, provides all the needed information to be able to understand what is contained within it. Furthermore, these files reside in a directory hierarchy which identifies what devices the traffic belongs to, the date the capture file was created. Finally, capture files reside in a directory which identifies where in the sequence of capture of a given day it was created. 
+Automation recipes expanding the range of data types collected can just follow this convention. This ensures interoperability and findability between various capture methods.
+
+\cref{ch4} \ref{sec:add-dev} already showed examples of the naming convention when adding devices.
+
+
+\section{\cref{req:fair_preservation}: Data Preservation Practices}
+\textbf{Status: Partially Met} \\
+Specific data preservation practices are not taken. \iottb already follows the Library of Congress recommendations on data formats (see \citet{recommendedformatrsLOC}). Most data is stored in plain text, and the binary formats used are widely known within the field and there is no access barrier. 
+To enhance the testbeds' compliance with this requirement, automation recipes which back-up the data to secure locations periodically can be developed. The need for built-in preservation should be balanced with the goal of not introducing dependencies not related to the core aim of automated collection and FAIR storage. One way is just to have a repository of scripts which are not built in to the \iottb executable, but which users can  use and adapt to their  needs\footnote{For instance rsync scripts with predefined filters appropriate for the database.}.
+
+\section{\cref{req:fair_accessibility}: Accessibility Controls}
+\textbf{Status: x} \\
+While the \iottb executable is ware what data it can and cannot access or change, there are currently no  wider access controls implemented. 
+
+

thesis/Chapters/ch6-conclusion.tex

@@ -0,0 +1,7 @@
+\chapter{Conclusion}\label{ch:conclusion}
+\iottbsc is an attempt for at an automation testbed for \iot devices.
+The \iottb package can be considered somewhat feature limited and incomplete for a proper testbed, but it provides a foundation on which to build a more fully fledged system.
+\iottb currently automates the setup and configuration of network packet capture and saves relevant database.
+The testbed uses the file system as a database such that it is also navigable by humans, not just machines.
+Data is stored in a predictably named hierarchy, and files which are produced as a result of operating \iottb are both uniquely identifiable and interpretable for humans. This is achieved by using the file system paths to provide some context, such that file names must only contain minimal information to make it meaningful to humans. Additionally, all created resources are identified by a \uuid which ensures that even if data is accidentally moved, their data is linked at least in principle.
+In summary, \iottbsc is a testbed which takes the first step toward a future where data is FAIR and experiments are reproducible.

thesis/Front/Abstract.tex

@@ -0,0 +1,5 @@
+% !TEX root = ../Thesis.tex
+\chapter{Abstract}
+To systematically study and assess the privacy and security implications of IoTdevices, it is crucial to have a reliable method for conducting experiments and extracting meaningful data in a reproducible manner. This necessitates the development of a system —referred to as a "testbed"— that includes all the necessary tools, definitions, and automated environment setup required for conduction reproducible experiments on IoT devices.
+
+In this project, I aim to design and implement a testbed that automates and standardizes the collection and processing of network data from IoT devices. The outcome of this project is a Python package that facilitates these tasks, providing a foundation for reproducible IoT device experiments.

thesis/Front/Acknowledgment.tex

@@ -0,0 +1,3 @@
+% !TEX root = ../Thesis.tex
+\chapter{Acknowledgments}
+So Long, and Thanks for All the Fish. And the template.

thesis/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Ivan Giangreco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thesis/Makefile

@@ -0,0 +1,27 @@
+#
+#  Thanks to Kevin Urban for providing this makefile.
+#  Jul 2012
+#
+
+# choose which latex compiler you want to use (xelatex or pdflatex)
+TEX = pdflatex
+
+
+################
+# stop editing #
+################
+latexfile = Thesis
+
+# rerun pdf generation until it doesn't say rerun anymore
+$(latexfile).pdf: $(latexfile).bbl
+	while ($(TEX) $(latexfile); grep -q "Rerun to get cross" $(latexfile).log); do true; done
+
+$(latexfile).bbl: $(latexfile).aux
+	bibtex $(latexfile)
+
+$(latexfile).aux: $(latexfile).tex
+	$(TEX) $(latexfile)
+
+clean:
+	rm *.aux
+	rm *.bbl

thesis/README.md

@@ -0,0 +1 @@
+Latex Template for Bachelor and Master thesis (University of Basel, Department of Mathematics and Computer Science, DBIS)