Hopefully successfully integrate proper repo.

This commit is contained in:
Sebastian Lenzlinger
2024-06-30 00:02:59 +02:00
parent 01954bd5a6
commit d9d3f66fc8
30 changed files with 1497 additions and 14 deletions

View File

@@ -0,0 +1,89 @@
import json
import click
from pathlib import Path
import logging
import re
from iottb import definitions
from iottb.models.device_metadata import DeviceMetadata
from iottb.models.iottb_config import IottbConfig
from iottb.definitions import CFG_FILE_PATH, TB_ECHO_STYLES
logger = logging.getLogger(__name__)
def add_device_guided(ctx, cn, db):
click.echo('TODO: Implement')
logger.info('Adding device interactively')
#logger.debug(f'Parameters: {params}. value: {value}')
@click.command('add-device', help='Add a device to a database')
@click.option('--dev', '--device-name', type=str, required=True,
help='The name of the device to be added. If this string contains spaces or other special characters \
normalization is performed to derive a canonical name')
@click.option('--db', '--database', type=click.Path(exists=True, file_okay=False, dir_okay=True),
envvar='IOTTB_DB', show_envvar=True,
help='Database in which to add this device. If not specified use default from config.')
@click.option('--guided', is_flag=True, default=False, show_default=True, envvar='IOTTB_GUIDED_ADD', show_envvar=True,
help='Add device interactively')
def add_device(dev, db, guided):
"""Add a new device to a database
Device name must be supplied unless in an interactive setup. Database is taken from config by default.
"""
logger.info('add-device invoked')
# Step 1: Load Config
# Dependency: Config file must exist
config = IottbConfig(Path(CFG_FILE_PATH))
logger.debug(f'Config loaded: {config}')
# Step 2: Load database
# dependency: Database folder must exist
if db:
database = db
path = config.db_path_dict[database]
logger.debug(f'Resolved (path, db) {path}, {database}')
else:
path = config.default_db_location
database = config.default_database
logger.debug(f'Default (path, db) {path}, {database}')
click.secho(f'Using database {database}')
full_db_path = Path(path) / database
if not full_db_path.is_dir():
logger.warning(f'No database at {database}')
click.echo(f'Could not find a database.')
click.echo(f'You need to initialize the testbed before before you add devices!')
click.echo(f'To initialize the testbed in the default location run "iottb init-db"')
click.echo('Exiting...')
exit()
# Step 3: Check if device already exists in database
# dependency: DeviceMetadata object
device_metadata = DeviceMetadata(device_name=dev)
device_dir = full_db_path / device_metadata.canonical_name
# Check if device is already registered
if device_dir.exists():
logger.warning(f'Device directory {device_dir} already exists.')
click.echo(f'Device {dev} already exists in the database.')
click.echo('Exiting...')
exit()
try:
device_dir.mkdir()
except OSError as e:
logger.error(f'Error trying to create device {e}')
click.echo('Exiting...')
exit()
# Step 4: Save metadata into device_dir
metadata_path = device_dir / definitions.DEVICE_METADATA_FILE_NAME
with metadata_path.open('w') as metadata_file:
json.dump(device_metadata.__dict__, metadata_file, indent=4)
click.echo(f'Successfully added device {dev} to database')
logger.debug(f'Added device {dev} to database {database}. Full path of metadata {metadata_path}')
logger.info(f'Metadata for {dev} {device_metadata.print_attributes()}')

View File

@@ -0,0 +1,130 @@
from pathlib import Path
import logging
import click
from iottb import tb_echo
from iottb.definitions import DB_NAME, CFG_FILE_PATH
from iottb.models.iottb_config import IottbConfig
logger = logging.getLogger(__name__)
@click.group('util')
def tb():
pass
@click.command()
@click.option('--file', default=DB_NAME)
@click.option('--table', type=str, default='DefaultDatabase')
@click.option('--key')
@click.option('--value')
@click.pass_context
def set_key_in_table_to(ctx, file, table, key, value):
"""Edit config or metadata files. TODO: Implement"""
click.echo(f'set_key_in_table_to invoked')
logger.warning("Unimplemented subcommand invoked.")
@click.command()
@click.confirmation_option(prompt="Are you certain that you want to delete the cfg file?")
def rm_cfg():
""" Removes the cfg file from the filesystem.
This is mostly a utility during development. Once non-standard database locations are implemented,
deleting this would lead to iottb not being able to find them anymore.
"""
Path(CFG_FILE_PATH).unlink()
click.echo(f'Iottb configuration removed at {CFG_FILE_PATH}')
@click.command()
@click.confirmation_option(prompt="Are you certain that you want to delete the databases file?")
def rm_dbs(dbs):
""" Removes ALL(!) databases from the filesystem if they're empty.
Development utility currently unfit for use.
"""
config = IottbConfig()
paths = config.get_know_database_paths()
logger.debug(f'Known db paths: {str(paths)}')
for dbs in paths:
try:
Path(dbs).rmdir()
click.echo(f'{dbs} deleted')
except Exception as e:
logger.debug(f'Failed unlinking db {dbs} with error {e}')
logger.info(f'All databases deleted')
@click.command('show-cfg', help='Show the current configuration context')
@click.option('--cfg-file', type=click.Path(), default=CFG_FILE_PATH, help='Path to the config file')
@click.option('-pp', is_flag=True, default=False, help='Pretty Print')
@click.pass_context
def show_cfg(ctx, cfg_file, pp):
logger.debug(f'Pretty print option set to {pp}')
if pp:
try:
config = IottbConfig(Path(cfg_file))
click.echo("Configuration Context:")
click.echo(f"Default Database: {config.default_database}")
click.echo(f"Default Database Path: {config.default_db_location}")
click.echo("Database Locations:")
for db_name, db_path in config.db_path_dict.items():
click.echo(f" - {db_name}: {db_path}")
except Exception as e:
logger.error(f"Error loading configuration: {e}")
click.echo(f"Failed to load configuration from {cfg_file}")
else:
path = Path(cfg_file)
if path.is_file():
with path.open('r') as file:
content = file.read()
click.echo(content)
else:
click.echo(f"Configuration file not found at {cfg_file}")
@click.command('show-all', help='Show everything: configuration, databases, and device metadata')
@click.pass_context
def show_everything(ctx):
"""Show everything that can be recursively found based on config except file contents."""
config = ctx.obj['CONFIG']
click.echo("Configuration Context:")
click.echo(f"Default Database: {config.default_database}")
click.echo(f"Default Database Path: {config.default_db_location}")
click.echo("Database Locations:")
everything_dict = {}
for db_name, db_path in config.db_path_dict.items():
click.echo(f" - {db_name}: {db_path}")
for db_name, db_path in config.db_path_dict.items():
full_db_path = Path(db_path) / db_name
if full_db_path.is_dir():
click.echo(f"\nContents of {full_db_path}:")
flag = True
for item in full_db_path.iterdir():
flag = False
if item.is_file():
click.echo(f" - {item.name}")
try:
with item.open('r', encoding='utf-8') as file:
content = file.read()
click.echo(f" Content:\n{content}")
except UnicodeDecodeError:
click.echo(" Content is not readable as text")
elif item.is_dir():
click.echo(f" - {item.name}/")
for subitem in item.iterdir():
if subitem.is_file():
click.echo(f" - {subitem.name}")
elif subitem.is_dir():
click.echo(f" - {subitem.name}/")
if flag:
tb_echo(f'\t EMPTY')
else:
click.echo(f"{full_db_path} is not a directory")

View File

@@ -0,0 +1,327 @@
import os
import shutil
import uuid
from time import time
import click
import subprocess
import json
from pathlib import Path
import logging
import re
from datetime import datetime
from click_option_group import optgroup
from iottb.definitions import APP_NAME, CFG_FILE_PATH
from iottb.models.iottb_config import IottbConfig
from iottb.utils.string_processing import make_canonical_name
# Setup logger
logger = logging.getLogger('iottb.sniff')
def is_ip_address(address):
ip_pattern = re.compile(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$")
return ip_pattern.match(address) is not None
def is_mac_address(address):
mac_pattern = re.compile(r"^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$")
return mac_pattern.match(address) is not None
def load_config(cfg_file):
"""Loads configuration from the given file path."""
with open(cfg_file, 'r') as config_file:
return json.load(config_file)
def validate_sniff(ctx, param, value):
logger.info('Validating sniff...')
if ctx.params.get('unsafe') and not value:
return None
if not ctx.params.get('unsafe') and not value:
raise click.BadParameter('Address is required unless --unsafe is set.')
if not is_ip_address(value) and not is_mac_address(value):
raise click.BadParameter('Address must be a valid IP address or MAC address.')
return value
@click.command('sniff', help='Sniff packets with tcpdump')
@optgroup.group('Testbed sources')
@optgroup.option('--db', '--database', type=str, envvar='IOTTB_DB', show_envvar=True,
help='Database of device. Only needed if not current default.')
@optgroup.option('--app', type=str, help='Companion app being used during capture', required=False)
@optgroup.group('Runtime behaviour')
@optgroup.option('--unsafe', is_flag=True, default=False, envvar='IOTTB_UNSAFE', is_eager=True,
help='Disable checks for otherwise required options.\n', show_envvar=True)
@optgroup.option('--guided', is_flag=True, default=False, envvar='IOTTB_GUIDED', show_envvar=True)
@optgroup.option('--pre', type=click.Path(exists=True, executable=True), help='Script to be executed before main '
'command'
'is started.')
@optgroup.group('Tcpdump options')
@optgroup.option('-i', '--interface',
help='Network interface to capture on.' +
'If not specified tcpdump tries to find and appropriate one.\n', show_envvar=True,
envvar='IOTTB_CAPTURE_INTERFACE')
@optgroup.option('-a', '--address', callback=validate_sniff,
help='IP or MAC address to filter packets by.\n', show_envvar=True,
envvar='IOTTB_CAPTURE_ADDRESS')
@optgroup.option('-I', '--monitor-mode', help='Put interface into monitor mode.', is_flag=True)
@optgroup.option('--ff', type=str, envvar='IOTTB_CAPTURE_FILTER', show_envvar=True,
help='tcpdump filter as string or file path.')
@optgroup.option('-#', '--print-pacno', is_flag=True, default=True,
help='Print packet number at beginning of line. True by default.')
@optgroup.option('-e', '--print-ll', is_flag=True, default=False,
help='Print link layer headers. True by default.')
@optgroup.option('-c', '--count', type=int, help='Number of packets to capture.', default=1000)
# @optgroup.option('--mins', type=int, help='Time in minutes to capture.', default=1)
@click.argument('tcpdump-args', nargs=-1, required=False, metavar='[TCPDUMP-ARGS]')
@click.argument('device', required=False)
@click.pass_context
def sniff(ctx, device, interface, print_pacno, ff, count, monitor_mode, print_ll, address, db, unsafe, guided,
app, tcpdump_args, **params):
""" Sniff packets from a device """
logger.info('sniff command invoked')
# Step1: Load Config
config = ctx.obj['CONFIG']
logger.debug(f'Config loaded: {config}')
# Step2: determine relevant database
database = db if db else config.default_database
path = config.db_path_dict[database]
full_db_path = Path(path) / database
logger.debug(f'Full db path is {str(full_db_path)}')
# 2.2: Check if it exists
if not full_db_path.is_dir():
logger.error('DB unexpectedly missing')
click.echo('DB unexpectedly missing')
return
canonical_name, aliases = make_canonical_name(device)
click.echo(f'Using canonical device name {canonical_name}')
device_path = full_db_path / canonical_name
# Step 3: now the device
if not device_path.exists():
if not unsafe:
logger.error(f'Device path {device_path} does not exist')
click.echo(f'Device path {device_path} does not exist')
return
else:
device_path.mkdir(parents=True, exist_ok=True)
logger.info(f'Device path {device_path} created')
click.echo(f'Found device at path {device_path}')
# Step 4: Generate filter
generic_filter = None
cap_filter = None
if ff:
logger.debug(f'ff: {ff}')
if Path(ff).is_file():
logger.info('Given filter option is a file')
with open(ff, 'r') as f:
cap_filter = f.read().strip()
else:
logger.info('Given filter option is an expression')
cap_filter = ff
else:
if address is not None:
if is_ip_address(address):
generic_filter = 'net'
cap_filter = f'{generic_filter} {address}'
elif is_mac_address(address):
generic_filter = 'ether net'
cap_filter = f'{generic_filter} {address}'
elif not unsafe:
logger.error('Invalid address format')
click.echo('Invalid address format')
return
logger.info(f'Generic filter {generic_filter}')
click.echo(f'Using filter {cap_filter}')
# Step 5: prep capture directory
capture_date = datetime.now().strftime('%Y-%m-%d')
capture_base_dir = device_path / f'sniffs/{capture_date}'
capture_base_dir.mkdir(parents=True, exist_ok=True)
logger.debug(f'Previous captures {capture_base_dir.glob('cap*')}')
capture_count = sum(1 for _ in capture_base_dir.glob('cap*'))
logger.debug(f'Capture count is {capture_count}')
capture_dir = f'cap{capture_count:04d}-{datetime.now().strftime('%H%M')}'
logger.debug(f'capture_dir: {capture_dir}')
# Full path
capture_dir_full_path = capture_base_dir / capture_dir
capture_dir_full_path.mkdir(parents=True, exist_ok=True)
click.echo(f'Files will be placed in {str(capture_dir_full_path)}')
logger.debug(f'successfully created capture directory')
# Step 6: Prepare capture file names
# Generate UUID for filenames
capture_uuid = str(uuid.uuid4())
click.echo(f'Capture has id {capture_uuid}')
pcap_file = f"{canonical_name}_{capture_uuid}.pcap"
pcap_file_full_path = capture_dir_full_path / pcap_file
stdout_log_file = f'stdout_{capture_uuid}.log'
stderr_log_file = f'stderr_{capture_uuid}.log'
logger.debug(f'Full pcap file path is {pcap_file_full_path}')
logger.info(f'pcap file name is {pcap_file}')
logger.info(f'stdout log file is {stdout_log_file}')
logger.info(f'stderr log file is {stderr_log_file}')
# Step 7: Build tcpdump command
logger.debug(f'pgid {os.getpgrp()}')
logger.debug(f'ppid {os.getppid()}')
logger.debug(f'(real, effective, saved) user id: {os.getresuid()}')
logger.debug(f'(real, effective, saved) group id: {os.getresgid()}')
cmd = ['sudo', 'tcpdump']
# 7.1 process flags
flags = []
if print_pacno:
flags.append('-#')
if print_ll:
flags.append('-e')
if monitor_mode:
flags.append('-I')
flags.append('-n') # TODO: Integrate, in case name resolution is wanted!
cmd.extend(flags)
flags_string = " ".join(flags)
logger.debug(f'Flags: {flags_string}')
# debug interlude
verbosity = ctx.obj['VERBOSITY']
if verbosity > 0:
verbosity_flag = '-'
for i in range(0, verbosity):
verbosity_flag = verbosity_flag + 'v'
logger.debug(f'verbosity string to pass to tcpdump: {verbosity_flag}')
cmd.append(verbosity_flag)
# 7.2 generic (i.e. reusable) kw args
generic_kw_args = []
if count:
generic_kw_args.extend(['-c', str(count)])
# if mins:
# generic_kw_args.extend(['-G', str(mins * 60)]) TODO: this currently loads to errors with sudo
cmd.extend(generic_kw_args)
generic_kw_args_string = " ".join(generic_kw_args)
logger.debug(f'KW args: {generic_kw_args_string}')
# 7.3 special kw args (not a priori reusable)
non_generic_kw_args = []
if interface:
non_generic_kw_args.extend(['-i', interface])
non_generic_kw_args.extend(['-w', str(pcap_file_full_path)])
cmd.extend(non_generic_kw_args)
non_generic_kw_args_string = " ".join(non_generic_kw_args)
logger.debug(f'Non transferable (special) kw args: {non_generic_kw_args_string}')
# 7.4 add filter expression
if cap_filter:
logger.debug(f'cap_filter (not generic): {cap_filter}')
cmd.append(cap_filter)
full_cmd_string = " ".join(cmd)
logger.info(f'tcpdump command: {"".join(full_cmd_string)}')
click.echo('Capture setup complete!')
# Step 8: Execute tcpdump command
start_time = datetime.now().strftime("%H:%M:%S")
start = time()
try:
if guided:
click.confirm(f'Execute following command: {full_cmd_string}')
stdout_log_file_abs_path = capture_dir_full_path / stdout_log_file
stderr_log_file_abs_path = capture_dir_full_path / stderr_log_file
stdout_log_file_abs_path.touch(mode=0o777)
stderr_log_file_abs_path.touch(mode=0o777)
with open(stdout_log_file_abs_path, 'w') as out, open(stderr_log_file_abs_path, 'w') as err:
logger.debug(f'\nstdout: {out}.\nstderr: {err}.\n')
tcp_complete = subprocess.run(cmd, check=True, capture_output=True, text=True)
out.write(tcp_complete.stdout)
err.write(tcp_complete.stderr)
#click.echo(f'Mock sniff execution')
click.echo(f"Capture complete. Saved to {pcap_file}")
except subprocess.CalledProcessError as e:
logger.error(f'Failed to capture packets: {e}')
click.echo(f'Failed to capture packets: {e}')
click.echo(f'Check {stderr_log_file} for more info.')
if ctx.obj['DEBUG']:
msg = [f'STDERR log {stderr_log_file} contents:\n']
with open(capture_dir_full_path / stderr_log_file) as log:
for line in log:
msg.append(line)
click.echo("\t".join(msg), lvl='e')
# print('DEBUG ACTIVE')
if guided:
click.prompt('Create metadata anyway?')
else:
click.echo('Aborting capture...')
exit()
end_time = datetime.now().strftime("%H:%M:%S")
end = time()
delta = end - start
click.echo(f'tcpdump took {delta:.2f} seconds.')
# Step 9: Register metadata
metadata = {
'device': canonical_name,
'device_id': device,
'capture_id': capture_uuid,
'capture_date_iso': datetime.now().isoformat(),
'invoked_command': " ".join(map(str, cmd)),
'capture_duration': delta,
'generic_parameters': {
'flags': flags_string,
'kwargs': generic_kw_args_string,
'filter': generic_filter
},
'non_generic_parameters': {
'kwargs': non_generic_kw_args_string,
'filter': cap_filter
},
'features': {
'interface': interface,
'address': address
},
'resources': {
'pcap_file': str(pcap_file),
'stdout_log': str(stdout_log_file),
'stderr_log': str(stderr_log_file)
},
'environment': {
'capture_dir': capture_dir,
'database': database,
'capture_base_dir': str(capture_base_dir),
'capture_dir_abs_path': str(capture_dir_full_path)
}
}
click.echo('Ensuring correct ownership of created files.')
username = os.getlogin()
gid = os.getgid()
# Else there are issues when running with sudo:
try:
subprocess.run(f'sudo chown -R {username}:{username} {device_path}', shell=True)
except OSError as e:
click.echo(f'Some error {e}')
click.echo(f'Saving metadata.')
metadata_abs_path = capture_dir_full_path / 'capture_metadata.json'
with open(metadata_abs_path, 'w') as f:
json.dump(metadata, f, indent=4)
click.echo(f'END SNIFF SUBCOMMAND')

View File

@@ -0,0 +1,120 @@
import click
from pathlib import Path
import logging
from logging.handlers import RotatingFileHandler
import sys
from iottb.models.iottb_config import IottbConfig
from iottb.definitions import DB_NAME, CFG_FILE_PATH
logger = logging.getLogger(__name__)
@click.command()
@click.option('-d', '--dest', type=click.Path(), help='Location to put (new) iottb database')
@click.option('-n', '--name', default=DB_NAME, type=str, help='Name of new database.')
@click.option('--update-default/--no-update-default', default=True, help='If new db should be set as the new default')
@click.pass_context
def init_db(ctx, dest, name, update_default):
logger.info('init-db invoked')
config = ctx.obj['CONFIG']
logger.debug(f'str(config)')
# Use the default path from config if dest is not provided
known_dbs = config.get_known_databases()
logger.debug(f'Known databases: {known_dbs}')
if name in known_dbs:
dest = config.get_database_location(name)
if Path(dest).joinpath(name).is_dir():
click.echo(f'A database {name} already exists.')
logger.debug(f'DB {name} exists in {dest}')
click.echo(f'Exiting...')
exit()
logger.debug(f'DB name {name} registered but does not exist.')
if not dest:
logger.info('No dest set, choosing default destination.')
dest = Path(config.default_db_location).parent
db_path = Path(dest).joinpath(name)
logger.debug(f'Full path for db {str(db_path)}')
# Create the directory if it doesn't exist
db_path.mkdir(parents=True, exist_ok=True)
logger.info(f"mkdir {db_path} successful")
click.echo(f'Created {db_path}')
# Update configuration
config.set_database_location(name, str(dest))
if update_default:
config.set_default_database(name, str(dest))
config.save_config()
logger.info(f"Updated configuration with database {name} at {db_path}")
# @click.group('config')
# @click.pass_context
# def cfg(ctx):
# pass
#
# @click.command('set', help='Set the location of a database.')
# @click.argument('database', help='Name of database')
# @click.argument('location', help='Where the database is located (i.e. its parent directory)')
# @click.pass_context
# def set(ctx, key, value):
# click.echo(f'Setting {key} to {value} in config')
# config = ctx.obj['CONFIG']
# logger.warning('No checks performed!')
# config.set_database_location(key, value)
# config.save_config()
@click.command()
@click.option('-d', '--dest', type=click.Path(), help='Location to put (new) iottb database')
@click.option('-n', '--name', default=DB_NAME, type=str, help='Name of new database.')
@click.option('--update-default/--no-update-default', default=True, help='If new db should be set as the new default')
@click.pass_context
def init_db_inactive(ctx, dest, name, update_default):
logger.info('init-db invoked')
config = ctx.obj['CONFIG']
logger.debug(f'str(config)')
# Retrieve known databases
known_dbs = config.get_known_databases()
# Determine destination path
if name in known_dbs:
dest = Path(config.get_database_location(name))
if dest.joinpath(name).is_dir():
click.echo(f'A database {name} already exists.')
logger.debug(f'DB {name} exists in {dest}')
click.echo(f'Exiting...')
exit()
logger.debug(f'DB name {name} registered but does not exist.')
elif not dest:
logger.info('No destination set, using default path from config.')
dest = Path(config.default_db_location).parent
# Ensure destination path is absolute
dest = dest.resolve()
# Combine destination path with database name
db_path = dest / name
logger.debug(f'Full path for database: {str(db_path)}')
# Create the directory if it doesn't exist
try:
db_path.mkdir(parents=True, exist_ok=True)
logger.info(f'Directory {db_path} created successfully.')
click.echo(f'Created {db_path}')
except Exception as e:
logger.error(f'Failed to create directory {db_path}: {e}')
click.echo(f'Failed to create directory {db_path}: {e}', err=True)
exit(1)
# Update configuration
config.set_database_location(name, str(db_path))
if update_default:
config.set_default_database(name, str(db_path))
config.save_config()
logger.info(f'Updated configuration with database {name} at {db_path}')
click.echo(f'Updated configuration with database {name} at {db_path}')