diff --git a/analysis/db_connector.py b/analysis/db_connector.py index 90dc740..c3f3706 100644 --- a/analysis/db_connector.py +++ b/analysis/db_connector.py @@ -1,9 +1,12 @@ import logging + +import sqlalchemy from sshtunnel import SSHTunnelForwarder from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from config import SSH_HOST, SSH_USERNAME, SSH_PASSWORD, DB_NAME, DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, SSH_PORT + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger('db_connector.py') stream_handler = logging.StreamHandler() @@ -11,6 +14,7 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) + class RemoteDB: def __init__(self): self.ssh_host = SSH_HOST @@ -47,7 +51,7 @@ class RemoteDB: def execute_query(self, query): session = self.Session() try: - result = session.execute(query) + result = session.execute(sqlalchemy.text(query)) session.commit() return result.fetchall() except Exception as e: diff --git a/analysis/map.py b/analysis/map.py index e6cebca..d2add2c 100644 --- a/analysis/map.py +++ b/analysis/map.py @@ -4,6 +4,7 @@ import os import folium from folium import plugins import logging +from db_connector import RemoteDB logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger('map.py') @@ -26,6 +27,9 @@ fixed_map_zurich_original_coords = folium.Map( scrollWheelZoom=False, doubleClickZoom=False ) + +def create_heat_view(): + pass def create_acc_map(): acc_gdf = gpd.read_file(accidents_filepath) acc_gdf['latitude'] = acc_gdf.geometry.y diff --git a/analysis/tester.ipynb b/analysis/tester.ipynb new file mode 100644 index 0000000..7542efc --- /dev/null +++ b/analysis/tester.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-01-04T15:00:59.587867376Z", + "start_time": "2024-01-04T15:00:59.470683932Z" + } + }, + "outputs": [], + "source": [ + "from db_connector import RemoteDB" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 16:00:59,621 - paramiko.transport - DEBUG - starting thread (client mode): 0xefbb5010\n", + "2024-01-04 16:00:59,622 - paramiko.transport - DEBUG - Local version/idstring: SSH-2.0-paramiko_3.4.0\n", + "2024-01-04 16:00:59,654 - paramiko.transport - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.6\n", + "2024-01-04 16:00:59,655 - paramiko.transport - INFO - Connected (version 2.0, client OpenSSH_8.9p1)\n", + "2024-01-04 16:00:59,685 - paramiko.transport - DEBUG - === Key exchange possibilities ===\n", + "2024-01-04 16:00:59,685 - paramiko.transport - DEBUG - kex algos: curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, sntrup761x25519-sha512@openssh.com, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, kex-strict-s-v00@openssh.com\n", + "2024-01-04 16:00:59,686 - paramiko.transport - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519\n", + "2024-01-04 16:00:59,687 - paramiko.transport - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com\n", + "2024-01-04 16:00:59,687 - paramiko.transport - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com\n", + "2024-01-04 16:00:59,688 - paramiko.transport - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1\n", + "2024-01-04 16:00:59,688 - paramiko.transport - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1\n", + "2024-01-04 16:00:59,688 - paramiko.transport - DEBUG - client compress: none, zlib@openssh.com\n", + "2024-01-04 16:00:59,689 - paramiko.transport - DEBUG - server compress: none, zlib@openssh.com\n", + "2024-01-04 16:00:59,689 - paramiko.transport - DEBUG - client lang: \n", + "2024-01-04 16:00:59,689 - paramiko.transport - DEBUG - server lang: \n", + "2024-01-04 16:00:59,690 - paramiko.transport - DEBUG - kex follows: False\n", + "2024-01-04 16:00:59,690 - paramiko.transport - DEBUG - === Key exchange agreements ===\n", + "2024-01-04 16:00:59,690 - paramiko.transport - DEBUG - Strict kex mode: True\n", + "2024-01-04 16:00:59,691 - paramiko.transport - DEBUG - Kex: curve25519-sha256@libssh.org\n", + "2024-01-04 16:00:59,691 - paramiko.transport - DEBUG - HostKey: ssh-ed25519\n", + "2024-01-04 16:00:59,691 - paramiko.transport - DEBUG - Cipher: aes128-ctr\n", + "2024-01-04 16:00:59,691 - paramiko.transport - DEBUG - MAC: hmac-sha2-256\n", + "2024-01-04 16:00:59,691 - paramiko.transport - DEBUG - Compression: none\n", + "2024-01-04 16:00:59,692 - paramiko.transport - DEBUG - === End of kex handshake ===\n", + "2024-01-04 16:00:59,765 - paramiko.transport - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode\n", + "2024-01-04 16:00:59,766 - paramiko.transport - DEBUG - kex engine KexCurve25519 specified hash_algo \n", + "2024-01-04 16:00:59,767 - paramiko.transport - DEBUG - Switch to new keys ...\n", + "2024-01-04 16:00:59,768 - paramiko.transport - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode\n", + "2024-01-04 16:00:59,769 - paramiko.transport - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,sk-ssh-ed25519@openssh.com,ssh-rsa,rsa-sha2-256,rsa-sha2-512,ssh-dss,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ecdsa-sha2-nistp256@openssh.com,webauthn-sk-ecdsa-sha2-nistp256@openssh.com', 'publickey-hostbound@openssh.com': b'0'}\n", + "2024-01-04 16:00:59,769 - paramiko.transport - DEBUG - Attempting public-key auth...\n", + "2024-01-04 16:00:59,859 - paramiko.transport - DEBUG - userauth is OK\n", + "2024-01-04 16:00:59,903 - paramiko.transport - INFO - Authentication (publickey) failed.\n", + "2024-01-04 16:00:59,931 - paramiko.transport - DEBUG - starting thread (client mode): 0xefbb7c50\n", + "2024-01-04 16:00:59,932 - paramiko.transport - DEBUG - Local version/idstring: SSH-2.0-paramiko_3.4.0\n", + "2024-01-04 16:00:59,969 - paramiko.transport - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.6\n", + "2024-01-04 16:00:59,970 - paramiko.transport - INFO - Connected (version 2.0, client OpenSSH_8.9p1)\n", + "2024-01-04 16:00:59,994 - paramiko.transport - DEBUG - === Key exchange possibilities ===\n", + "2024-01-04 16:00:59,995 - paramiko.transport - DEBUG - kex algos: curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, sntrup761x25519-sha512@openssh.com, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, kex-strict-s-v00@openssh.com\n", + "2024-01-04 16:00:59,996 - paramiko.transport - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519\n", + "2024-01-04 16:00:59,996 - paramiko.transport - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com\n", + "2024-01-04 16:00:59,996 - paramiko.transport - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com\n", + "2024-01-04 16:00:59,997 - paramiko.transport - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1\n", + "2024-01-04 16:00:59,997 - paramiko.transport - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1\n", + "2024-01-04 16:00:59,998 - paramiko.transport - DEBUG - client compress: none, zlib@openssh.com\n", + "2024-01-04 16:00:59,998 - paramiko.transport - DEBUG - server compress: none, zlib@openssh.com\n", + "2024-01-04 16:00:59,998 - paramiko.transport - DEBUG - client lang: \n", + "2024-01-04 16:00:59,999 - paramiko.transport - DEBUG - server lang: \n", + "2024-01-04 16:00:59,999 - paramiko.transport - DEBUG - kex follows: False\n", + "2024-01-04 16:01:00,000 - paramiko.transport - DEBUG - === Key exchange agreements ===\n", + "2024-01-04 16:01:00,000 - paramiko.transport - DEBUG - Strict kex mode: True\n", + "2024-01-04 16:01:00,000 - paramiko.transport - DEBUG - Kex: curve25519-sha256@libssh.org\n", + "2024-01-04 16:01:00,000 - paramiko.transport - DEBUG - HostKey: ssh-ed25519\n", + "2024-01-04 16:01:00,001 - paramiko.transport - DEBUG - Cipher: aes128-ctr\n", + "2024-01-04 16:01:00,001 - paramiko.transport - DEBUG - MAC: hmac-sha2-256\n", + "2024-01-04 16:01:00,001 - paramiko.transport - DEBUG - Compression: none\n", + "2024-01-04 16:01:00,002 - paramiko.transport - DEBUG - === End of kex handshake ===\n", + "2024-01-04 16:01:00,004 - paramiko.transport - DEBUG - EOF in transport thread\n", + "2024-01-04 16:01:00,089 - paramiko.transport - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode\n", + "2024-01-04 16:01:00,090 - paramiko.transport - DEBUG - kex engine KexCurve25519 specified hash_algo \n", + "2024-01-04 16:01:00,090 - paramiko.transport - DEBUG - Switch to new keys ...\n", + "2024-01-04 16:01:00,090 - paramiko.transport - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode\n", + "2024-01-04 16:01:00,091 - paramiko.transport - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,sk-ssh-ed25519@openssh.com,ssh-rsa,rsa-sha2-256,rsa-sha2-512,ssh-dss,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ecdsa-sha2-nistp256@openssh.com,webauthn-sk-ecdsa-sha2-nistp256@openssh.com', 'publickey-hostbound@openssh.com': b'0'}\n", + "2024-01-04 16:01:00,091 - paramiko.transport - DEBUG - Attempting public-key auth...\n", + "2024-01-04 16:01:00,192 - paramiko.transport - DEBUG - userauth is OK\n", + "2024-01-04 16:01:00,244 - paramiko.transport - INFO - Authentication (publickey) successful!\n" + ] + } + ], + "source": [ + "remote_db = RemoteDB()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-04T15:01:00.276638120Z", + "start_time": "2024-01-04T15:00:59.590634120Z" + } + }, + "id": "62c8f38132d5cd27" + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-04 16:01:00,277 - paramiko.transport - DEBUG - [chan 0] Max packet in: 32768 bytes\n", + "2024-01-04 16:01:00,334 - paramiko.transport - DEBUG - Received global request \"hostkeys-00@openssh.com\"\n", + "2024-01-04 16:01:00,334 - paramiko.transport - DEBUG - Rejecting \"hostkeys-00@openssh.com\" global request from server.\n", + "2024-01-04 16:01:00,335 - paramiko.transport - DEBUG - Debug msg: b'/home/sebl/.ssh/authorized_keys:4: key options: agent-forwarding port-forwarding pty user-rc x11-forwarding'\n", + "2024-01-04 16:01:00,358 - paramiko.transport - DEBUG - [chan 0] Max packet out: 32768 bytes\n", + "2024-01-04 16:01:00,359 - paramiko.transport - DEBUG - Secsh channel 0 opened.\n", + "2024-01-04 16:01:01,360 - paramiko.transport - DEBUG - [chan 0] EOF sent (0)\n", + "2024-01-04 16:01:01,361 - paramiko.transport - DEBUG - [chan 1] Max packet in: 32768 bytes\n", + "2024-01-04 16:01:01,386 - paramiko.transport - DEBUG - [chan 0] EOF received (0)\n", + "2024-01-04 16:01:01,409 - paramiko.transport - DEBUG - [chan 1] Max packet out: 32768 bytes\n", + "2024-01-04 16:01:01,409 - paramiko.transport - DEBUG - Secsh channel 1 opened.\n" + ] + } + ], + "source": [ + "result = remote_db.execute_query(\"select * from footbikecount where id = 1\")" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-04T15:01:02.003242832Z", + "start_time": "2024-01-04T15:01:00.277738067Z" + } + }, + "id": "29b60614b47dbc87" + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [ + { + "data": { + "text/plain": "[(1, 1250443, 2678956, '2012-01-01', 0, 0, 0, 0, 0, 'Sunday')]" + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-04T15:01:10.779405322Z", + "start_time": "2024-01-04T15:01:10.773113980Z" + } + }, + "id": "cddcbd64b8c52d93" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}