X-Git-Url: https://www.yuggoth.org/gitweb?p=weather.git;a=blobdiff_plain;f=weather.py;h=bdda3ee821527e7d59acb41f40141a7d8145fb9b;hp=78fd79c58375a0dbf43be90644a4377df87b14b6;hb=8ad1f3f02c90003035fd6dbeb558752a1bb9cb0b;hpb=8a37eddc0667410e77de9d1441d90532440a0329

diff --git a/weather.py b/weather.py
index 78fd79c..bdda3ee 100644
--- a/weather.py
+++ b/weather.py
@@ -212,7 +212,7 @@ def get_uri(
             except (IOError, OSError): pass
         dcache_fn = os.path.join(
             dcachedir,
-            uri.split(":")[1].replace("/","_")
+            uri.split(":",1)[1].replace("/","_")
         )
     now = time.time()
     if cache_data and os.access(dcache_fn, os.R_OK) \
@@ -222,8 +222,7 @@ def get_uri(
         dcache_fd.close()
     else:
         try:
-            if pyversion("3"): data = urlopen(uri).read().decode("utf-8")
-            else: data = urlopen(uri).read()
+            data = urlopen(uri).read().decode("utf-8")
         except URLError:
             if ignore_fail: return ""
             else:
@@ -238,6 +237,9 @@ def get_uri(
                     )
                 sys.stderr.write(message)
                 sys.exit(1)
+        # Some data sources are HTML with the plain text wrapped in pre tags
+        if "<pre>" in data:
+            data = data[data.find("<pre>")+5:data.find("</pre>")]
         if cache_data:
             try:
                 import codecs
@@ -1103,7 +1105,7 @@ def guess(
             print(
                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
             )
-        elif searchtype is "coordinates":
+        elif searchtype == "coordinates":
             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
         if zone[0]:
             print(
@@ -1119,7 +1121,7 @@ def guess(
             print(
                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
             )
-        elif searchtype is "coordinates" and zone[0]:
+        elif searchtype == "coordinates" and zone[0]:
             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
     if cache_search:
         now = time.time()
@@ -1132,7 +1134,7 @@ def guess(
         )
         search_cache = ["\n"]
         search_cache.append( "[%s]\n" % search[0] ) 
-        search_cache.append( "description = cached %s\n" % nowstamp )
+        search_cache.append( "cached = %s\n" % nowstamp )
         for uriname in sorted(uris.keys()):
             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
         real_cachedir = os.path.expanduser(cachedir)
@@ -1266,56 +1268,56 @@ def correlate():
         weather_copyright,
         os.path.basename( sys.argv[0] ),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( time.time() )
+            datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
         ),
         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gcounties_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
         ),
         gcounties_an,
         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
         ),
         gcousubs_an,
         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gplace_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
         ),
         gplace_an,
         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
         ),
         gzcta_an,
         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(cpfzcf_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
         ),
         cpfzcf_fn,
         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
         ),
         nsdcccc_fn,
         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(ourairports_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
         ),
         ourairports_fn,
         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
         ),
         overrides_fn,
         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(slist_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
         ),
         slist_fn,
         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
         datetime.date.isoformat(
-            datetime.datetime.fromtimestamp( os.path.getmtime(zlist_fn) )
+            datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
         ),
         zlist_fn
     )
@@ -1328,7 +1330,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU")
+    gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
     columns = gcounties.readline().decode("utf-8").strip().split("\t")
     for line in gcounties:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1351,7 +1353,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU")
+    gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
     for line in gcousubs:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1374,7 +1376,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "rU")
+    gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
     columns = gplace.readline().decode("utf-8").strip().split("\t")
     for line in gplace:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1443,18 +1445,18 @@ def correlate():
     count = 0
     ourairports = open(ourairports_fn, "rU")
     for row in csv.reader(ourairports):
-        icao = row[12].decode('utf-8').lower()
+        icao = row[12].lower()
         if icao in stations:
-            iata = row[13].decode('utf-8').lower()
+            iata = row[13].lower()
             if len(iata) == 3: airports[iata] = { "station": icao }
             if "description" not in stations[icao]:
                 description = []
-                name = row[3].decode('utf-8')
+                name = row[3]
                 if name: description.append(name)
-                municipality = row[10].decode('utf-8')
+                municipality = row[10]
                 if municipality: description.append(municipality)
-                region = row[9].decode('utf-8')
-                country = row[8].decode('utf-8')
+                region = row[9]
+                country = row[8]
                 if region:
                     if "-" in region:
                         c,r = region.split("-", 1)
@@ -1465,9 +1467,9 @@ def correlate():
                 if description:
                     stations[icao]["description"] = ", ".join(description)
             if "location" not in stations[icao]:
-                lat = row[4].decode('utf-8')
+                lat = row[4]
                 if lat:
-                    lon = row[5].decode('utf-8')
+                    lon = row[5]
                     if lon:
                         stations[icao]["location"] = gecos(
                             "%s,%s" % (lat, lon)
@@ -1564,7 +1566,7 @@ def correlate():
     sys.stdout.write(message)
     sys.stdout.flush()
     count = 0
-    gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU")
+    gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
     columns = gzcta.readline().decode("utf-8").strip().split("\t")
     for line in gzcta:
         fields = line.decode("utf-8").strip().split("\t")
@@ -1737,6 +1739,12 @@ def correlate():
     if os.path.exists(overrideslog_fn):
         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
+    import time
+    overrideslog_fd.write(
+        '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
+        '# use, copy, modify, and distribute this software is granted under terms\n'
+        '# provided in the LICENSE file distributed with this software.\n\n'
+        % time.gmtime().tm_year)
     overrideslog_fd.writelines(overrideslog)
     overrideslog_fd.close()
     print("done (%s overridden sections: +%s/-%s/!%s)." % (
@@ -1947,6 +1955,8 @@ def correlate():
                     if type(element) is float: elements.append("%.7f"%element)
                     else: elements.append( repr(element) )
                 value = "(%s)"%", ".join(elements)
+            if type(value) is bytes:
+                value = value.decode("utf-8")
             stations_fd.write( "\n%s = %s" % (key, value) )
         count += 1
     stations_fd.write("\n")
@@ -2085,6 +2095,12 @@ def correlate():
     if os.path.exists(qalog_fn):
         os.rename(qalog_fn, "%s_old"%qalog_fn)
     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
+    import time
+    qalog_fd.write(
+        '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
+        '# use, copy, modify, and distribute this software is granted under terms\n'
+        '# provided in the LICENSE file distributed with this software.\n\n'
+        % time.gmtime().tm_year)
     qalog_fd.writelines(qalog)
     qalog_fd.close()
     if qalog: