Repairing stuff

api/api.py

@@ -48,7 +48,7 @@ def index():
     else:
         if 'id' in request.args:
             station_id = request.args['id']
-            wheres = wheres + (sql.SQL("AND id = {values} ").format(column=sql.Identifier('stations', 'id'), values=sql.Placeholder()))
+            wheres = wheres + (sql.SQL("AND station_id = {values} ").format(column=sql.Identifier('stations', 'station_id'), values=sql.Placeholder()))
             for n in [int(station_id)]:
                 values = (*values, n)  # adding n to existing tuple
 
@@ -61,7 +61,7 @@ def index():
 
         # Composing query
         query = sql.SQL("SELECT array_to_json(array_agg(row_to_json(t))) from ("
-                        "SELECT id, {} FROM stations "
+                        "SELECT station_id, {} FROM stations "
                         "WHERE lon IS NOT NULL "    # Unnecessary filter, just so the real filters can always be written with AND
                         "{} "
                         ") t;").format(columns, wheres)

dataacquisition/ExportToDatabase.py

@@ -13,10 +13,10 @@ import psycopg2.extras
 from psycopg2 import sql
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 import configparser
-#from api.GetAverageData import get_interpolation_data_for_point
+from api.GetAverageData import get_interpolation_data_for_point
 
 cfg = configparser.ConfigParser()
-cfg.read('./config.ini')
+cfg.read('../config.ini')
 assert "POSTGRES" in cfg, "missing POSTGRES in config.ini"
 param_postgres = cfg["POSTGRES"]
 param_interpol = cfg["INTERPOLATION"]
@@ -73,7 +73,7 @@ def check_for_db_existence(cursor):
 def create_table(station_list, cursor):
     print('Creating table stations')
     df_columns = list(station_list)
-    columns = ['id INTEGER', 'lon NUMERIC', 'lat NUMERIC', 'country TEXT', 'file TEXT']
+    columns = ['station_id INTEGER', 'lon NUMERIC', 'lat NUMERIC', 'country TEXT', 'file TEXT']
     for column in df_columns:
         if str(column).startswith('19') or str(column).startswith('20'):
             columns.append('"{}" NUMERIC'.format(column))
@@ -104,7 +104,7 @@ def insert_empty_matrix_into_db(cursor):
             for n in [id, lon, lat]:
                 values = (*values, n)  # adding n to existing tuple
 
-            query = sql.SQL("INSERT INTO STATIONS (id, lon, lat, country) "
+            query = sql.SQL("INSERT INTO STATIONS (station_id, lon, lat, country) "
                             "VALUES ({id}, {lon}, {lat}, 'Germany');").format(id=sql.Placeholder(), lon=sql.Placeholder(), lat=sql.Placeholder())
             # print(query.as_string(cursor))
             # print(values)
@@ -117,7 +117,7 @@ def insert_empty_matrix_into_db(cursor):
 def create_matrix_data(cursor, amount_points):
     print('Calculating interpolation data for matrix')
     # start_time = time.time()
-    cursor.execute("SELECT id, lon, lat FROM stations WHERE file is NULL;")
+    cursor.execute("SELECT station_id, lon, lat FROM stations WHERE file is NULL;")
     matrix_points = cursor.fetchall()
     update_data = []
     for i, point in enumerate(matrix_points):
@@ -133,7 +133,7 @@ def create_matrix_data(cursor, amount_points):
             print(round(finished), end="% ... ")
     print('', end=' 100%, Done. \n')
 
-    query = sql.SQL("""UPDATE stations SET "%(year)s" = %(value)s WHERE id = %(id)s; """)
+    query = sql.SQL("""UPDATE stations SET "%(year)s" = %(value)s WHERE station_id = %(id)s; """)
     print('Writing interpolation data to database')
     psycopg2.extras.execute_batch(cursor, query, update_data)   # Multiple times faster than using execute() in a for loop
     # print((time.time() - start_time), 'seconds')
@@ -143,16 +143,20 @@ def create_matrix_data(cursor, amount_points):
 # Dumping all existing data from database. Inserting station data into database in bulk.
 def createInsertStatement(station_list):
     # create INSERT INTO table (columns) VALUES('%s',...)
-    station_list.columns.astype(str)
-    
+    # station_list.columns.astype(str)
+
     df_columns = list(station_list)
-    station_list = station_list.round(decimals=3)
+    columns = ['"' + column + '"' for column in df_columns]
+    columns = str(columns).replace('[', '').replace(']', '').replace("'", "").replace('\n', '').replace(' ', '')
     values = "VALUES({})".format(",".join(["%s" for _ in df_columns]))
+    df_columns = str(df_columns).strip('[]')
+    station_list = station_list.round(decimals=3)
     # create INSERT INTO table (columns) VALUES('%s',...)
-    insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', df_columns, values)
+    insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', columns, values)
     
     return insert_stmt
 
+
 def insert_data(station_list, cursor):
     print('Inserting data into database')
     if len(station_list) > 0:
@@ -177,7 +181,7 @@ def insert_data(station_list, cursor):
         # create INSERT INTO table (columns) VALUES('%s',...)
         #insert_stmt = """INSERT INTO {} ({}) {}""".format('stations', columns, values)
         insert_stmt = createInsertStatement(station_list)
-        
+        print(insert_stmt)
         psycopg2.extras.execute_batch(cursor, insert_stmt, station_list.values)
     print('Done')
 

dataacquisition/sandbox.py

-import time
 
-top = 101
-
-for i in range(0, top):
-    time.sleep(0.1)
-    if i % 10 == 0:
-        finished = i / top * 100
-
-        print(round(finished), end="% ... ")
+query = """'lon', 'lat', 'file', 'country', '2018', '2017', '2016', '2015', '2014', '2013', '2012', '2011', '2010', '2009', '2008', '2007', '2006', '2005', '2004', '2003', '2002', '2001', '2000', '1999', '1998', '1997', '1996', '1995', '1994', '1993', '1992', '1991', '1990', '1989', '1988', '1987', '1986', '1985', '1984', '1983', '1982', '1981', '1980', '1979', '1978', '1977', '1976', '1975', '1974', '1973', '1972', '1971', '1970', '1969', '1968', '1967', '1966', '1965', '1964', '1963', '1962', '1961', '1960', '1959', '1958', '1957', '1956', '1955', '1954', '1953', '1952', '1951', '1950', '1949', 'station_id') VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"""
+print(len(query.split('%s')))
\ No newline at end of file