From 2e3c5b83dc0b6093f1fb597850761e92fefe9cd6 Mon Sep 17 00:00:00 2001
From: Clemens Berteld <clemens@berteld.com>
Date: Mon, 20 Sep 2021 17:07:49 +0200
Subject: [PATCH] Repairing stuff
---
api/api.py | 4 ++--
dataacquisition/ExportToDatabase.py | 26 +++++++++++++++-----------
dataacquisition/sandbox.py | 11 ++---------
3 files changed, 19 insertions(+), 22 deletions(-)
diff --git a/api/api.py b/api/api.py
index 0f0fa09..1b7587e 100644
--- a/api/api.py
+++ b/api/api.py
@@ -48,7 +48,7 @@ def index():
else:
if 'id' in request.args:
station_id = request.args['id']
- wheres = wheres + (sql.SQL("AND id = {values} ").format(column=sql.Identifier('stations', 'id'), values=sql.Placeholder()))
+ wheres = wheres + (sql.SQL("AND station_id = {values} ").format(column=sql.Identifier('stations', 'station_id'), values=sql.Placeholder()))
for n in [int(station_id)]:
values = (*values, n) # adding n to existing tuple
@@ -61,7 +61,7 @@ def index():
# Composing query
query = sql.SQL("SELECT array_to_json(array_agg(row_to_json(t))) from ("
- "SELECT id, {} FROM stations "
+ "SELECT station_id, {} FROM stations "
"WHERE lon IS NOT NULL " # Unnecessary filter, just so the real filters can always be written with AND
"{} "
") t;").format(columns, wheres)
diff --git a/dataacquisition/ExportToDatabase.py b/dataacquisition/ExportToDatabase.py
index cb82bc0..e017479 100644
--- a/dataacquisition/ExportToDatabase.py
+++ b/dataacquisition/ExportToDatabase.py
@@ -13,10 +13,10 @@ import psycopg2.extras
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import configparser
-#from api.GetAverageData import get_interpolation_data_for_point
+from api.GetAverageData import get_interpolation_data_for_point
cfg = configparser.ConfigParser()
-cfg.read('./config.ini')
+cfg.read('../config.ini')
assert "POSTGRES" in cfg, "missing POSTGRES in config.ini"
param_postgres = cfg["POSTGRES"]
param_interpol = cfg["INTERPOLATION"]
@@ -73,7 +73,7 @@ def check_for_db_existence(cursor):
def create_table(station_list, cursor):
print('Creating table stations')
df_columns = list(station_list)
- columns = ['id INTEGER', 'lon NUMERIC', 'lat NUMERIC', 'country TEXT', 'file TEXT']
+ columns = ['station_id INTEGER', 'lon NUMERIC', 'lat NUMERIC', 'country TEXT', 'file TEXT']
for column in df_columns:
if str(column).startswith('19') or str(column).startswith('20'):
columns.append('"{}" NUMERIC'.format(column))
@@ -104,7 +104,7 @@ def insert_empty_matrix_into_db(cursor):
for n in [id, lon, lat]:
values = (*values, n) # adding n to existing tuple
- query = sql.SQL("INSERT INTO STATIONS (id, lon, lat, country) "
+ query = sql.SQL("INSERT INTO STATIONS (station_id, lon, lat, country) "
"VALUES ({id}, {lon}, {lat}, 'Germany');").format(id=sql.Placeholder(), lon=sql.Placeholder(), lat=sql.Placeholder())
# print(query.as_string(cursor))
# print(values)
@@ -117,7 +117,7 @@ def insert_empty_matrix_into_db(cursor):
def create_matrix_data(cursor, amount_points):
print('Calculating interpolation data for matrix')
# start_time = time.time()
- cursor.execute("SELECT id, lon, lat FROM stations WHERE file is NULL;")
+ cursor.execute("SELECT station_id, lon, lat FROM stations WHERE file is NULL;")
matrix_points = cursor.fetchall()
update_data = []
for i, point in enumerate(matrix_points):
@@ -133,7 +133,7 @@ def create_matrix_data(cursor, amount_points):
print(round(finished), end="% ... ")
print('', end=' 100%, Done. \n')
- query = sql.SQL("""UPDATE stations SET "%(year)s" = %(value)s WHERE id = %(id)s; """)
+ query = sql.SQL("""UPDATE stations SET "%(year)s" = %(value)s WHERE station_id = %(id)s; """)
print('Writing interpolation data to database')
psycopg2.extras.execute_batch(cursor, query, update_data) # Multiple times faster than using execute() in a for loop
# print((time.time() - start_time), 'seconds')
@@ -143,16 +143,20 @@ def create_matrix_data(cursor, amount_points):
# Dumping all existing data from database. Inserting station data into database in bulk.
def createInsertStatement(station_list):
# create INSERT INTO table (columns) VALUES('%s',...)
- station_list.columns.astype(str)
-
+ # station_list.columns.astype(str)
+
df_columns = list(station_list)
- station_list = station_list.round(decimals=3)
+ columns = ['"' + column + '"' for column in df_columns]
+ columns = str(columns).replace('[', '').replace(']', '').replace("'", "").replace('\n', '').replace(' ', '')
values = "VALUES({})".format(",".join(["%s" for _ in df_columns]))
+ df_columns = str(df_columns).strip('[]')
+ station_list = station_list.round(decimals=3)
# create INSERT INTO table (columns) VALUES('%s',...)
- insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', df_columns, values)
+ insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', columns, values)
return insert_stmt
+
def insert_data(station_list, cursor):
print('Inserting data into database')
if len(station_list) > 0:
@@ -177,7 +181,7 @@ def insert_data(station_list, cursor):
# create INSERT INTO table (columns) VALUES('%s',...)
#insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', columns, values)
insert_stmt = createInsertStatement(station_list)
-
+ print(insert_stmt)
psycopg2.extras.execute_batch(cursor, insert_stmt, station_list.values)
print('Done')
diff --git a/dataacquisition/sandbox.py b/dataacquisition/sandbox.py
index 4fff5af..da3ffa8 100644
--- a/dataacquisition/sandbox.py
+++ b/dataacquisition/sandbox.py
@@ -1,10 +1,3 @@
-import time
-top = 101
-
-for i in range(0, top):
- time.sleep(0.1)
- if i % 10 == 0:
- finished = i / top * 100
-
- print(round(finished), end="% ... ")
+query = """'lon', 'lat', 'file', 'country', '2018', '2017', '2016', '2015', '2014', '2013', '2012', '2011', '2010', '2009', '2008', '2007', '2006', '2005', '2004', '2003', '2002', '2001', '2000', '1999', '1998', '1997', '1996', '1995', '1994', '1993', '1992', '1991', '1990', '1989', '1988', '1987', '1986', '1985', '1984', '1983', '1982', '1981', '1980', '1979', '1978', '1977', '1976', '1975', '1974', '1973', '1972', '1971', '1970', '1969', '1968', '1967', '1966', '1965', '1964', '1963', '1962', '1961', '1960', '1959', '1958', '1957', '1956', '1955', '1954', '1953', '1952', '1951', '1950', '1949', 'station_id') VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"""
+print(len(query.split('%s')))
\ No newline at end of file
--
GitLab