Remove sqlite support from codebase

.github/workflows/CI.yml

@@ -129,10 +129,13 @@ jobs:
           maturin develop --release --extras=test,docs
           python -m pip install --upgrade pytest pytest-dotenv pytest-cov furl hypothesis trustme astor pyopenssl isort pydirectory
 
+      - name: Create testdata directory
+        run: mkdir -p ./AISdb/testdata
+
       - name: Run Tests
         run: |
           AISdb\Scripts\activate
-          pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --maxfail=10
+           pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --ignore=./aisdb/tests/test_001_postgres.py --maxfail=10
 
       - name: Clean Up
         run: |
@@ -186,6 +189,7 @@ jobs:
           database: postgres
           postgres-version: "17"
           ssl: false
+          port: 5432
         id: postgres
 
       - name: Test PostgreSQL Connection
@@ -205,11 +209,6 @@ jobs:
         run: |
           sudo apt-get install -y timescaledb-2-postgresql-17 postgresql-client-17
           echo "TimescaleDB Extension installed"
-
-      # - name: Apply TimescaleDB tuning
-      #   run: |
-      #     sudo timescaledb-tune --pg-config="/home/runner/work/_temp/pgdata/postgresql.conf" --quiet --yes
-      #     echo "TimescaleDB tuning applied"
 
       - name: Enable TimescaleDB in PostgreSQL Config
         run: |
@@ -237,10 +236,13 @@ jobs:
           maturin develop --release --extras=test,docs
           python -m pip install --upgrade pytest pytest-dotenv pytest-cov furl hypothesis trustme astor pyopenssl isort pydirectory
 
+      - name: Create testdata directory
+        run: mkdir -p ./AISdb/testdata
+
       - name: Run Tests
         run: |
           source ./AISdb/bin/activate
-          pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --maxfail=10
+          pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --ignore=./aisdb/tests/test_001_postgres.py --maxfail=10
 
       - name: Clean Up
         run: |
@@ -350,10 +352,13 @@ jobs:
           maturin develop --release --extras=test,docs
           python -m pip install --upgrade pytest pytest-dotenv pytest-cov furl hypothesis trustme astor pyopenssl isort pydirectory
 
+      - name: Create testdata directory
+        run: mkdir -p ./AISdb/testdata
+
       - name: Run Tests
         run: |
           source ./AISdb/bin/activate
-          pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --maxfail=10
+          pytest ./aisdb/tests/ --ignore=./aisdb/tests/test_014_marinetraffic.py --ignore=./aisdb/tests/test_001_postgres.py --maxfail=10
 
       - name: Clean Up
         run: |
@@ -591,4 +596,4 @@ jobs:
   #     - name: Clean Up
   #       run: |
   #         rm -rf gh-pages
-  #         rm -rf docs/dist_sphinx
+  #         rm -rf docs/dist_sphinx

aisdb/__init__.py

@@ -9,21 +9,13 @@
                      'pyproject.toml'), 'r') as tomlfile:
     __version__ = toml.load(tomlfile).get('project').get('version')
 
-import sqlite3
-
-if (sqlite3.sqlite_version_info[0] < 3
-        or (sqlite3.sqlite_version_info[0] <= 3
-            and sqlite3.sqlite_version_info[1] < 8)):
-    warnings.warn(
-        f"An outdated version of SQLite was found ({sqlite3.sqlite_version})")
-
 sqlpath = os.path.abspath(os.path.join(os.path.dirname(__file__), 'aisdb_sql'))
 
 import aisdb.web_interface
 
 from .database.decoder import decode_msgs
 
-from .database.dbconn import DBConn, SQLiteDBConn, PostgresDBConn
+from .database.dbconn import DBConn, PostgresDBConn
 
 from .database.dbqry import DBQuery
 

aisdb/aisdb_sql/createtable_static_global_aggregate.sql

@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS static_global_aggregate (
+    mmsi INTEGER PRIMARY KEY,
+    imo INTEGER,
+    vessel_name TEXT,
+    ship_type INTEGER,
+    call_sign TEXT,
+    dim_bow INTEGER,
+    dim_stern INTEGER,
+    dim_port INTEGER,
+    dim_star INTEGER,
+    draught INTEGER,
+    destination TEXT,
+    eta_month INTEGER,
+    eta_day INTEGER,
+    eta_hour INTEGER,
+    eta_minute INTEGER
+);

aisdb/aisdb_sql/cte_aliases_global.sql

@@ -0,0 +1,6 @@
+dynamic_global AS (
+    {}
+),
+static_global AS (
+    {}
+)

aisdb/aisdb_sql/cte_dynamic_clusteredidx_global.sql

@@ -0,0 +1,13 @@
+SELECT 
+    d.mmsi, 
+    d.time,
+    d.utc_second,
+    d.longitude,
+    d.latitude,
+    d.rot,
+    d.sog,
+    d.cog,
+    d.heading,
+    d.maneuver
+FROM ais_global_dynamic AS d
+WHERE

aisdb/aisdb_sql/cte_static_aggregate_global.sql

@@ -0,0 +1,16 @@
+SELECT 
+    agg.mmsi, 
+    TRIM(agg.vessel_name) as vessel_name, 
+    agg.ship_type,
+    agg.imo,
+    agg.dim_bow,
+    agg.dim_stern,
+    agg.dim_port,
+    agg.dim_star,
+    agg.draught,
+    agg.destination,
+    agg.eta_month,
+    agg.eta_day,
+    agg.eta_hour,
+    agg.eta_minute
+FROM static_global_aggregate AS agg

aisdb/aisdb_sql/new_insert_dynamic_clusteredidx.sql

@@ -0,0 +1,16 @@
+INSERT INTO ais_global_dynamic
+(
+    mmsi,
+    time,
+    longitude,
+    latitude,
+    rot,
+    sog,
+    cog,
+    heading,
+    maneuver,
+    utc_second,
+    source
+)
+VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
+ON CONFLICT DO NOTHING;

aisdb/aisdb_sql/new_insert_static.sql

@@ -0,0 +1,23 @@
+INSERT INTO ais_global_static (
+    mmsi,
+    time,
+    vessel_name,
+    ship_type,
+    call_sign,
+    imo,
+    dim_bow,
+    dim_stern,
+    dim_port,
+    dim_star,
+    draught,
+    destination,
+    ais_version,
+    fixing_device,
+    eta_month,
+    eta_day,
+    eta_hour,
+    eta_minute,
+    source
+  )
+VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)
+ON CONFLICT DO NOTHING;

aisdb/aisdb_sql/select_join_dynamic_static_clusteredidx_global.sql

@@ -0,0 +1,28 @@
+SELECT 
+    dynamic.mmsi, 
+    dynamic.time,
+    dynamic.utc_second,
+    dynamic.longitude,
+    dynamic.latitude,
+    dynamic.rot,
+    dynamic.sog,
+    dynamic.cog,
+    dynamic.heading,
+    dynamic.maneuver,
+    static.imo,
+    static.ship_type,
+    static.vessel_name,
+    static.dim_bow,
+    static.dim_stern,
+    static.dim_port,
+    static.dim_star,
+    static.draught,
+    static.destination,
+    static.eta_month,
+    static.eta_day,
+    static.eta_hour,
+    static.eta_minute,
+    ref.coarse_type_txt AS ship_type_txt
+FROM ais_global_dynamic AS dynamic
+LEFT JOIN ais_global_static AS static ON dynamic.mmsi = static.mmsi
+LEFT JOIN ref ON static.ship_type = ref.coarse_type

aisdb/aisdb_sql/timescale_createtable_dynamic.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS ais_{0}_dynamic
+CREATE TABLE IF NOT EXISTS ais_global_dynamic
 (
     mmsi          INTEGER NOT NULL,
     time          INTEGER NOT NULL,
@@ -15,14 +15,14 @@ CREATE TABLE IF NOT EXISTS ais_{0}_dynamic
 );
 
 SELECT create_hypertable(
-    'ais_{0}_dynamic',
+    'ais_global_dynamic',
     'time',
     partitioning_column => 'mmsi',
     number_partitions => 4,
     chunk_time_interval => 604800
 );
 
-ALTER TABLE ais_{0}_dynamic SET (
+ALTER TABLE ais_global_dynamic SET (
     timescaledb.compress = false,
     timescaledb.compress_orderby = 'time ASC, latitude ASC, longitude ASC',
     timescaledb.compress_segmentby = 'mmsi'

aisdb/aisdb_sql/timescale_createtable_static.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS ais_{0}_static
+CREATE TABLE IF NOT EXISTS ais_global_static
 (
     mmsi           INTEGER NOT NULL,
     time           INTEGER NOT NULL,
@@ -23,14 +23,14 @@ CREATE TABLE IF NOT EXISTS ais_{0}_static
 );
 
 SELECT create_hypertable(
-        'ais_{0}_static',
+        'ais_global_static',
         'time',
         partitioning_column => 'mmsi',
         number_partitions => 4,
         chunk_time_interval => 604800
 );
 
-ALTER TABLE ais_{0}_static SET (
+ALTER TABLE ais_global_static SET (
     timescaledb.compress = false,
     timescaledb.compress_orderby = 'time ASC',
     timescaledb.compress_segmentby = 'mmsi'

aisdb/database/create_tables.py

@@ -11,3 +11,6 @@
 
 with open(os.path.join(sqlpath, 'createtable_static_aggregate.sql'), 'r') as f:
     sql_aggregate = f.read()
+
+with open(os.path.join(sqlpath, 'createtable_static_global_aggregate.sql'), 'r') as f:
+    sql_global_aggregate = f.read()