From 1ad33abf5b8e6aa0a33a8874dff6293e7cfd4496 Mon Sep 17 00:00:00 2001
From: Clemens Berteld <clemens@berteld.com>
Date: Wed, 15 Sep 2021 18:53:47 +0200
Subject: [PATCH] Weighted interpolation fully working now (lat and long were
mixed up)
---
dataacquisition/ExportToDatabase.py | 4 +--
dataacquisition/GetAverageData.py | 2 +-
dataacquisition/sandbox.py | 41 +++++++++++++++++++++++++----
3 files changed, 38 insertions(+), 9 deletions(-)
diff --git a/dataacquisition/ExportToDatabase.py b/dataacquisition/ExportToDatabase.py
index eaa5bb6..127698f 100644
--- a/dataacquisition/ExportToDatabase.py
+++ b/dataacquisition/ExportToDatabase.py
@@ -91,7 +91,7 @@ def create_table(station_list, db_name):
def insert_empty_matrix_into_db():
with psycopg2.connect(database=param_postgres['dbName'], user=param_postgres["user"], password=param_postgres["password"], host=param_postgres["host"], port=param_postgres["port"]) as connection:
with connection.cursor() as cursor:
- with open('clipped_matrix_10x10.csv', 'r') as matrix:
+ with open('clipped_matrix_25x25.csv', 'r') as matrix:
matrix_data = matrix.readlines()
for line in matrix_data[1:]:
values = '' # Used in second parameter of cursor.execute() (Avoids SQL injection)
@@ -121,13 +121,11 @@ def create_matrix_data():
id = point[0]
lon = point[1]
lat = point[2]
- print(lat, lon)
interpolation_data = get_interpolation_data_for_point(lat, lon, sql.SQL('*'))
for year, value in interpolation_data.items():
update_data.append({'year': year, 'value': value, 'id': id})
- print(update_data)
query = sql.SQL("""UPDATE stations SET "%(year)s" = %(value)s WHERE id = %(id)s; """)
psycopg2.extras.execute_batch(cursor, query, update_data) # 2 times faster than using execute() in a for loop, ~20 mins instead of 40
diff --git a/dataacquisition/GetAverageData.py b/dataacquisition/GetAverageData.py
index b1100f5..9f38717 100644
--- a/dataacquisition/GetAverageData.py
+++ b/dataacquisition/GetAverageData.py
@@ -31,7 +31,7 @@ def get_neighbours(cursor, lat, lon, columns):
query = sql.SQL("""
SELECT array_to_json(array_agg(row_to_json(t))) from (
- SELECT {columns}, ST_Distance(ST_MakePoint(lon, lat), ST_MakePoint({lon}, {lat})) AS distance
+ SELECT {columns}, ST_Distance(ST_MakePoint(lat, lon), ST_MakePoint({lon}, {lat})) AS distance
FROM stations
WHERE file IS NOT NULL
ORDER BY distance
diff --git a/dataacquisition/sandbox.py b/dataacquisition/sandbox.py
index e47664c..1cb8907 100644
--- a/dataacquisition/sandbox.py
+++ b/dataacquisition/sandbox.py
@@ -1,5 +1,14 @@
-# import psycopg2
-# from psycopg2 import sql
+import configparser
+
+import psycopg2
+from psycopg2 import sql
+cfg = configparser.ConfigParser()
+cfg.read('config.ini')
+
+assert "POSTGRES" in cfg, "missing POSTGRES in config.ini"
+
+param_postgres = cfg["POSTGRES"]
+param_interpol = cfg["INTERPOLATION"]
#
# query = sql.SQL("select {0} from {1} where {2} LIKE {3} {4}").format(sql.SQL(', ').join([sql.Identifier('foo'), sql.Identifier('bar')]), sql.Identifier('table'), sql.Identifier('coulumn'), sql.Placeholder(),
# sql.SQL('AND 1 = 1'))
@@ -20,8 +29,30 @@
# list_b = [x * 2 for x in list_a if x % 2 == 0]
# print(list_b)
+# list_a = [1,2,3,4,5,6,7,8,9,10]
+# new_list = list_a[0::2]
+# print(new_list)
+
+
+def get_neighbours(lat, lon, columns):
+ values = '' # Used in second parameter of cursor.execute() (Avoids SQL injection)
+ for n in [lat, lon]:
+ values = (*values, n) # adding n to existing tuple
+ with psycopg2.connect(database=param_postgres["dbName"], user=param_postgres["user"], password=param_postgres["password"], host=param_postgres["host"], port=param_postgres["port"]) as connection:
+ with connection.cursor() as cursor:
+ query = sql.SQL("""
+ SELECT array_to_json(array_agg(row_to_json(t))) from (
+ SELECT {columns}, ST_Distance(ST_MakePoint(lat, lon), ST_MakePoint({lon}, {lat})) AS distance
+ FROM stations
+ WHERE file IS NOT NULL
+ ORDER BY distance
+ LIMIT {amount_neighbours}
+ ) t;
+ """).format(columns=columns, lon=sql.Placeholder(), lat=sql.Placeholder(), amount_neighbours=sql.SQL(param_interpol["amount_neighbours"]))
+ cursor.execute(query, values)
+ neighbours = cursor.fetchall()[0][0]
+ print(neighbours)
+ # return neighbours
-list_a = [1,2,3,4,5,6,7,8,9,10]
-new_list = list_a[0::2]
-print(new_list)
\ No newline at end of file
+get_neighbours(53.42, 9.24, sql.SQL('"2015", "2014", "2010"'))
\ No newline at end of file
--
GitLab