adjust filter for germany exclude antarctica

dataacquisition/DwdAcquisition.py

@@ -4,12 +4,9 @@ Author: Peter Morstein
 
 import pandas as pd
 from ftplib import FTP
-import pickle
 import numpy as np
-import ExportToWorldShape as exportToWorldShape
+#import ExportToWorldShape as exportToWorldShape
 import ExportToDatabase as exportToDatabase
-from IPython.display import display
-from pip._internal.utils.misc import tabulate
 
 """ 
 example files:
@@ -34,28 +31,16 @@ def loadDWDGauges():
     stationList['lon'] = stationList['lon'].str.strip()
     stationList['lat'] = stationList['lat'].str.strip()
     
-    # rename countries to merge with geopandas world shape file
-    stationList.loc[stationList['country']=="Korea, Dem. People's Rep.", 'country'] = 'South Korea'
-    stationList.loc[stationList['country']=="Slovakia (Slovak. Rep.)", 'country'] = 'Slovakia'
-    stationList.loc[stationList['country']=="Slowenia", 'country'] = 'Slovenia'
-    stationList.loc[stationList['country']=="Russian Federation", 'country'] = 'Russia'
-    stationList.loc[stationList['country']=="Bosnia and Herzegowina", 'country'] = 'Bosnia and Herz.'
-    stationList.loc[stationList['country']=="Slovakia (Slovak. Rep.)", 'country'] = 'Slovakia'
-    stationList.loc[stationList['country']=="Croatia/Hrvatska", 'country'] = 'Croatia'
-    stationList.loc[stationList['country']=="Moldova, Rep. Of", 'country'] = 'Moldova'
-    stationList.loc[stationList['country']=="United Kingdom of Great Britain and N.-Ireland ", 'country'] = 'United Kingdom'
-    stationList.loc[stationList['country']=="Czech Republic", 'country'] = 'Czechia'
-    stationList.loc[stationList['country']=="Somalia", 'country'] = 'Somaliland'
-    stationList.loc[stationList['country']=="Iran (Islamic Rep. of)", 'country'] = 'Iran'
-    stationList.loc[stationList['country']=="Mauretania", 'country'] = 'Mauritania'
-    stationList.loc[stationList['country']=="Central African Republic", 'country'] = 'Central African Rep.'
-    stationList.loc[stationList['country']=="South Sudan", 'country'] = 'S. Sudan'
-    stationList.loc[stationList['country']=="Dem. Republic of the Congo", 'country'] = 'Dem. Rep. Congo'
-    stationList.loc[stationList['country']=="Mauretania", 'country'] = 'Somalia'
-    stationList.loc[stationList['country']=="Syrian Arab Rep.", 'country'] = 'Syria'
-    stationList.loc[stationList['country']=="Australien, SW-Pazifik", 'country'] = 'Australia'
-    stationList.loc[stationList['country']=="Western-Sahara",'country'] = "W. Sahara"
+    return stationList
+
+def filterDWDGauges(stationList):
+    stationList = stationList.loc[stationList['country'] == "Germany"]
+    
+    stationList['lat'] = pd.to_numeric(stationList['lat'], errors='coerce', downcast='float')
+    
+    stationList = stationList.loc[stationList['lat']>40]
     
+    return stationList
 
 # load station file names from DWD an join the filename with the stationList
 def loadAndJoinDWDClimateFilenames():
@@ -193,7 +178,7 @@ def start():
     print("___ DWD Acquisition start___")
     
     loadDWDGauges()
-    stationList = stationList.loc[stationList['country'] == "Germany"]
+    filterDWDGauges()
     loadAndJoinDWDClimateFilenames()
     loadTemperatureFromDWDGauges()
 
@@ -202,8 +187,6 @@ def start():
         # stationList = pickle.load(pickleFile)
         # stationList = pd.read_pickle('./pickle/stationList_germany.pickle')
 
-
-    
     # export station list to different outputs
     #exportToWorldShape.export(stationList)
 

dataacquisition/ExportToDatabase.py

@@ -13,6 +13,8 @@ import psycopg2.extras
 from psycopg2 import sql
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 import configparser
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from api.GetAverageData import get_interpolation_data_for_point
 
 cfg = configparser.ConfigParser()

dataacquisition/ExportToWorldShape.py

@@ -37,6 +37,31 @@ def cleanAverageTimeseries(stationList):
             stationList.at[stationID, "m1961T1990"] = None
             stationList.at[stationID, "m1991T2018"] = None
 
+def renameCountryNames(stationList):
+    
+    # rename countries to merge with geopandas world shape file
+    stationList.loc[stationList['country']=="Korea, Dem. People's Rep.", 'country'] = 'South Korea'
+    stationList.loc[stationList['country']=="Slovakia (Slovak. Rep.)", 'country'] = 'Slovakia'
+    stationList.loc[stationList['country']=="Slowenia", 'country'] = 'Slovenia'
+    stationList.loc[stationList['country']=="Russian Federation", 'country'] = 'Russia'
+    stationList.loc[stationList['country']=="Bosnia and Herzegowina", 'country'] = 'Bosnia and Herz.'
+    stationList.loc[stationList['country']=="Slovakia (Slovak. Rep.)", 'country'] = 'Slovakia'
+    stationList.loc[stationList['country']=="Croatia/Hrvatska", 'country'] = 'Croatia'
+    stationList.loc[stationList['country']=="Moldova, Rep. Of", 'country'] = 'Moldova'
+    stationList.loc[stationList['country']=="United Kingdom of Great Britain and N.-Ireland ", 'country'] = 'United Kingdom'
+    stationList.loc[stationList['country']=="Czech Republic", 'country'] = 'Czechia'
+    stationList.loc[stationList['country']=="Somalia", 'country'] = 'Somaliland'
+    stationList.loc[stationList['country']=="Iran (Islamic Rep. of)", 'country'] = 'Iran'
+    stationList.loc[stationList['country']=="Mauretania", 'country'] = 'Mauritania'
+    stationList.loc[stationList['country']=="Central African Republic", 'country'] = 'Central African Rep.'
+    stationList.loc[stationList['country']=="South Sudan", 'country'] = 'S. Sudan'
+    stationList.loc[stationList['country']=="Dem. Republic of the Congo", 'country'] = 'Dem. Rep. Congo'
+    stationList.loc[stationList['country']=="Mauretania", 'country'] = 'Somalia'
+    stationList.loc[stationList['country']=="Syrian Arab Rep.", 'country'] = 'Syria'
+    stationList.loc[stationList['country']=="Australien, SW-Pazifik", 'country'] = 'Australia'
+    stationList.loc[stationList['country']=="Western-Sahara",'country'] = "W. Sahara"
+    
+    return stationList
 
 def convertStationListToGPD(stationList):
     
@@ -47,6 +72,8 @@ def convertStationListToGPD(stationList):
     stationList = stationList[stationList.lat != ""]
     stationList = stationList[stationList.lon != ""]
     
+    stationList = renameCountryNames(stationList)
+    
     stationList["lat"] = stationList["lat"].astype(str).astype(float)
     stationList["lon"] = stationList["lon"].astype(str).astype(float)
     

dataacquisition/test_DwdAcquisition.py

@@ -13,6 +13,19 @@ import DwdAcquisition as cut
 
 class TestDwdAcquisition(unittest.TestCase):
     
+    def testFilterDWDGauges(self):
+        # given
+        stationList = cut.loadDWDGauges()
+        self.assertTrue(len(stationList)>1000)
+        
+        # test
+        stationList = cut.filterDWDGauges(stationList)
+        
+        # check
+        for i, station in stationList.iterrows():
+            self.assertTrue(station['lat']>40)
+        
+    
     def _testLoadTemperatureFromDWDGauges(self):
         global annualData
         cut.loadTemperatureFromDWDGauges()

dataacquisition/test_ExportToDatabase.py

@@ -152,7 +152,7 @@ class TestExportToDatabase(unittest.TestCase):
                 connection.close()
             self.assertRaises(error.message)
             print("Test FAILED__")
-        
+    
     def testCreateInsertStatement(self):
         print("__Test CreateInsertStatement")
         with open("./pickle/stationList_with_temperature.pickle", "rb") as pickleFile: