Refresh active station list and correlation data

[weather.git] / weather.py

diff --git a/weather.py b/weather.py
index 5b71fc0..b0c5855 100644 (file)
--- a/weather.py
+++ b/weather.py
@@ -89,14 +89,10 @@ class Selections:
                 return self.config.get(argument, option)
         if option in self.options.__dict__:
             return self.options.__dict__[option]
-        else:
-            import os, sys
-            message = "%s error: no URI defined for %s\n" % (
-                os.path.basename( sys.argv[0] ),
-                option
-            )
-            sys.stderr.write(message)
-            exit(1)
+        import sys
+        message = "WARNING: no URI defined for %s\n" % option
+        sys.stderr.write(message)
+        return None
     def get_bool(self, option, argument=None):
         """Get data and coerce to a boolean if necessary."""
         return bool(self.get(option, argument))
@@ -324,11 +320,7 @@ def get_alert(
 ):
     """Return alert notice for the specified URI."""
     if not uri:
-        import os, sys
-        message = "%s error: Alert URI required for alerts\n" % \
-            os.path.basename( sys.argv[0] )
-        sys.stderr.write(message)
-        sys.exit(1)
+        return ""
     alert = get_uri(
         uri,
         ignore_fail=True,
@@ -400,14 +392,9 @@ def get_options(config):
             + "flash_flood_watch," \
             + "flood_statement," \
             + "flood_warning," \
-            + "marine_weather_statement," \
-            + "river_statement," \
             + "severe_thunderstorm_warning," \
             + "severe_weather_statement," \
-            + "short_term_forecast," \
-            + "special_marine_warning," \
             + "special_weather_statement," \
-            + "tornado_warning," \
             + "urgent_weather_message"
     option_parser.add_option("--atypes",
         dest="atypes",
@@ -1268,56 +1255,56 @@ def correlate():
         weather_copyright,
         os.path.basename( sys.argv[0] ),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( time.time() )
+            datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
         ),
         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gcounties_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
         ),
         gcounties_an,
         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
         ),
         gcousubs_an,
         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gplace_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
         ),
         gplace_an,
         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
         ),
         gzcta_an,
         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(cpfzcf_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
         ),
         cpfzcf_fn,
         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
         ),
         nsdcccc_fn,
         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(ourairports_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
         ),
         ourairports_fn,
         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
         ),
         overrides_fn,
         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(slist_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
         ),
         slist_fn,
         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(zlist_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
         ),
         zlist_fn
     )
@@ -1330,7 +1317,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU")
+    gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
     columns = gcounties.readline().decode("utf-8").strip().split("\t")
     for line in gcounties:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1353,7 +1340,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU")
+    gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
     for line in gcousubs:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1376,7 +1363,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "rU")
+    gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
     columns = gplace.readline().decode("utf-8").strip().split("\t")
     for line in gplace:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1445,18 +1432,18 @@ def correlate():
     count = 0
     ourairports = open(ourairports_fn, "rU")
     for row in csv.reader(ourairports):
-        icao = row[12].decode('utf-8').lower()
+        icao = row[12].lower()
         if icao in stations:
-            iata = row[13].decode('utf-8').lower()
+            iata = row[13].lower()
             if len(iata) == 3: airports[iata] = { "station": icao }
             if "description" not in stations[icao]:
                 description = []
-                name = row[3].decode('utf-8')
+                name = row[3]
                 if name: description.append(name)
-                municipality = row[10].decode('utf-8')
+                municipality = row[10]
                 if municipality: description.append(municipality)
-                region = row[9].decode('utf-8')
-                country = row[8].decode('utf-8')
+                region = row[9]
+                country = row[8]
                 if region:
                     if "-" in region:
                         c,r = region.split("-", 1)
@@ -1467,9 +1454,9 @@ def correlate():
                 if description:
                     stations[icao]["description"] = ", ".join(description)
             if "location" not in stations[icao]:
-                lat = row[4].decode('utf-8')
+                lat = row[4]
                 if lat:
-                    lon = row[5].decode('utf-8')
+                    lon = row[5]
                     if lon:
                         stations[icao]["location"] = gecos(
                             "%s,%s" % (lat, lon)
@@ -1566,7 +1553,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU")
+    gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
     columns = gzcta.readline().decode("utf-8").strip().split("\t")
     for line in gzcta:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1955,6 +1942,8 @@ def correlate():
                     if type(element) is float: elements.append("%.7f"%element)
                     else: elements.append( repr(element) )
                 value = "(%s)"%", ".join(elements)
+            if type(value) is bytes:
+                value = value.decode("utf-8")
             stations_fd.write( "\n%s = %s" % (key, value) )
         count += 1
     stations_fd.write("\n")