+def correlate():
+ import codecs, datetime, hashlib, os, re, sys, tarfile, time, zipfile
+ if pyversion("3"): import configparser
+ else: import ConfigParser as configparser
+ gcounties_an = "2015_Gaz_counties_national.zip"
+ gcounties_fn = "2015_Gaz_counties_national.txt"
+ gcousubs_an = "2015_Gaz_cousubs_national.zip"
+ gcousubs_fn = "2015_Gaz_cousubs_national.txt"
+ gplace_an = "2015_Gaz_place_national.zip"
+ gplace_fn = "2015_Gaz_place_national.txt"
+ gzcta_an = "2015_Gaz_zcta_national.zip"
+ gzcta_fn = "2015_Gaz_zcta_national.txt"
+ for filename in os.listdir("."):
+ if re.match("bp[0-9][0-9][a-z][a-z][0-9][0-9].dbx$", filename):
+ cpfzcf_fn = filename
+ break
+ nsdcccc_fn = "nsd_cccc.txt"
+ zcatalog_an = "zonecatalog.curr.tar"
+ metartbl_fn = "metar.tbl"
+ coopstn_fn = "coop-stations.txt"
+ overrides_fn = "overrides.conf"
+ overrideslog_fn = "overrides.log"
+ slist_fn = "slist"
+ zlist_fn = "zlist"
+ qalog_fn = "qa.log"
+ airports_fn = "airports"
+ places_fn = "places"
+ stations_fn = "stations"
+ zctas_fn = "zctas"
+ zones_fn = "zones"
+ header = """\
+%s
+# generated by %s on %s from these public domain sources:
+#
+# http://www.census.gov/geo/maps-data/data/gazetteer2015.html
+# %s %s %s
+# %s %s %s
+# %s %s %s
+# %s %s %s
+#
+# http://www.weather.gov/geodata/catalog/wsom/html/cntyzone.htm
+# %s %s %s
+#
+# http://tgftp.nws.noaa.gov/data/nsd_cccc.txt
+# %s %s %s
+#
+# http://tgftp.nws.noaa.gov/data/zonecatalog.curr.tar
+# %s %s %s
+#
+# http://www.nco.ncep.noaa.gov/pmb/codes/nwprod/dictionaries/metar.tbl
+# %s %s %s
+#
+# http://www.ncdc.noaa.gov/homr/reports
+# %s %s %s
+#
+# ...and these manually-generated or hand-compiled adjustments:
+# %s %s %s
+# %s %s %s
+# %s %s %s\
+""" % (
+ weather_copyright,
+ os.path.basename( sys.argv[0] ),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( time.time() )
+ ),
+ hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(gcounties_an) )
+ ),
+ gcounties_an,
+ hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) )
+ ),
+ gcousubs_an,
+ hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(gplace_an) )
+ ),
+ gplace_an,
+ hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) )
+ ),
+ gzcta_an,
+ hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(cpfzcf_fn) )
+ ),
+ cpfzcf_fn,
+ hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) )
+ ),
+ nsdcccc_fn,
+ hashlib.md5( open(zcatalog_an, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(zcatalog_an) )
+ ),
+ zcatalog_an,
+ hashlib.md5( open(metartbl_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(metartbl_fn) )
+ ),
+ metartbl_fn,
+ hashlib.md5( open(coopstn_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(coopstn_fn) )
+ ),
+ coopstn_fn,
+ hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) )
+ ),
+ overrides_fn,
+ hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(slist_fn) )
+ ),
+ slist_fn,
+ hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
+ datetime.date.isoformat(
+ datetime.datetime.fromtimestamp( os.path.getmtime(zlist_fn) )
+ ),
+ zlist_fn
+ )
+ airports = {}
+ places = {}
+ stations = {}
+ zctas = {}
+ zones = {}
+ message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU")
+ columns = gcounties.readline().decode("latin1").strip().split("\t")
+ for line in gcounties:
+ fields = line.decode("latin1").strip().split("\t")
+ f_geoid = fields[ columns.index("GEOID") ].strip()
+ f_name = fields[ columns.index("NAME") ].strip()
+ f_usps = fields[ columns.index("USPS") ].strip()
+ f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
+ f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
+ if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
+ fips = "fips%s" % f_geoid
+ if fips not in places: places[fips] = {}
+ places[fips]["centroid"] = gecos(
+ "%s,%s" % (f_intptlat, f_intptlong)
+ )
+ places[fips]["description"] = "%s, %s" % (f_name, f_usps)
+ count += 1
+ gcounties.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU")
+ columns = gcousubs.readline().decode("latin1").strip().split("\t")
+ for line in gcousubs:
+ fields = line.decode("latin1").strip().split("\t")
+ f_geoid = fields[ columns.index("GEOID") ].strip()
+ f_name = fields[ columns.index("NAME") ].strip()
+ f_usps = fields[ columns.index("USPS") ].strip()
+ f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
+ f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
+ if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
+ fips = "fips%s" % f_geoid
+ if fips not in places: places[fips] = {}
+ places[fips]["centroid"] = gecos(
+ "%s,%s" % (f_intptlat, f_intptlong)
+ )
+ places[fips]["description"] = "%s, %s" % (f_name, f_usps)
+ count += 1
+ gcousubs.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s:%s..." % (gplace_an, gplace_fn)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "rU")
+ columns = gplace.readline().decode("latin1").strip().split("\t")
+ for line in gplace:
+ fields = line.decode("latin1").strip().split("\t")
+ f_geoid = fields[ columns.index("GEOID") ].strip()
+ f_name = fields[ columns.index("NAME") ].strip()
+ f_usps = fields[ columns.index("USPS") ].strip()
+ f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
+ f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
+ if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
+ fips = "fips%s" % f_geoid
+ if fips not in places: places[fips] = {}
+ places[fips]["centroid"] = gecos(
+ "%s,%s" % (f_intptlat, f_intptlong)
+ )
+ places[fips]["description"] = "%s, %s" % (f_name, f_usps)
+ count += 1
+ gplace.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % slist_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ slist = codecs.open(slist_fn, "rU")
+ for line in slist:
+ icao = line.split("#")[0].strip()
+ if icao:
+ stations[icao] = {
+ "metar": "http://tgftp.nws.noaa.gov/data/observations/"\
+ + "metar/decoded/%s.TXT" % icao.upper()
+ }
+ count += 1
+ slist.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % metartbl_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ metartbl = codecs.open(metartbl_fn, "rU")
+ for line in metartbl:
+ icao = line[:4].strip().lower()
+ if icao in stations:
+ description = []
+ name = " ".join(
+ line[16:48].replace("_", " ").strip().title().split()
+ )
+ if name: description.append(name)
+ st = line[49:51].strip()
+ if st: description.append(st)
+ cn = line[52:54].strip()
+ if cn: description.append(cn)
+ if description:
+ stations[icao]["description"] = ", ".join(description)
+ lat = line[55:60].strip()
+ if lat:
+ lat = int(lat)/100.0
+ lon = line[61:67].strip()
+ if lon:
+ lon = int(lon)/100.0
+ stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
+ count += 1
+ metartbl.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % nsdcccc_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ nsdcccc = codecs.open(nsdcccc_fn, "rU", "latin1")
+ for line in nsdcccc:
+ line = str(line)
+ fields = line.split(";")
+ icao = fields[0].strip().lower()
+ if icao in stations:
+ description = []
+ name = " ".join( fields[3].strip().title().split() )
+ if name: description.append(name)
+ st = fields[4].strip()
+ if st: description.append(st)
+ country = " ".join( fields[5].strip().title().split() )
+ if country: description.append(country)
+ if description:
+ stations[icao]["description"] = ", ".join(description)
+ lat, lon = fields[7:9]
+ if lat and lon:
+ stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
+ elif "location" not in stations[icao]:
+ lat, lon = fields[5:7]
+ if lat and lon:
+ stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
+ count += 1
+ nsdcccc.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % coopstn_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ coopstn = open(coopstn_fn)
+ for line in coopstn:
+ icao = line[33:37].strip().lower()
+ if icao in stations:
+ iata = line[22:26].strip().lower()
+ if len(iata) == 3: airports[iata] = { "station": icao }
+ if "description" not in stations[icao]:
+ description = []
+ name = " ".join( line[99:129].strip().title().split() )
+ if name: description.append(name)
+ st = line[59:61].strip()
+ if st: description.append(st)
+ country = " ".join( line[38:58].strip().title().split() )
+ if country: description.append(country)
+ if description:
+ stations[icao]["description"] = ", ".join(description)
+ if "location" not in stations[icao]:
+ lat = line[130:139].strip()
+ if lat:
+ lat = lat.replace(" ", "-")
+ lon = line[140:150].strip()
+ if lon:
+ lon = lon.replace(" ", "-")
+ stations[icao]["location"] = gecos(
+ "%s,%s" % (lat, lon)
+ )
+ count += 1
+ coopstn.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % zlist_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ zlist = codecs.open(zlist_fn, "rU")
+ for line in zlist:
+ line = line.split("#")[0].strip()
+ if line:
+ zones[line] = {}
+ count += 1
+ zlist.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s:*..." % zcatalog_an
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ zcatalog = tarfile.open(zcatalog_an)
+ for entry in zcatalog.getmembers():
+ if entry.isfile():
+ fnmatch = re.match(
+ r"([a-z]+z[0-9]+)\.txt$",
+ os.path.basename(entry.name)
+ )
+ if fnmatch:
+ zone = fnmatch.group(1)
+ if zone in zones:
+ data = zcatalog.extractfile(entry).readlines()
+ description = data[0].decode("ascii").strip()
+ zones[zone]["description"] = description
+ for line in data[1:]:
+ line = line.decode("latin1").strip()
+ urimatch = re.match("/webdocs/(.+):(.+) for ", line)
+ if urimatch:
+ uritype = urimatch.group(2).lower().replace(" ","_")
+ zones[zone][uritype] \
+ = "http://weather.noaa.gov/%s" \
+ % urimatch.group(1)
+ count += 1
+ zcatalog.close()
+ print("done (%s files)." % count)
+ message = "Reading %s..." % cpfzcf_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ cpfz = {}
+ cpfzcf = open(cpfzcf_fn)
+ for line in cpfzcf:
+ fields = line.strip().split("|")
+ if len(fields) == 11 \
+ and fields[0] and fields[1] and fields[9] and fields[10]:
+ zone = "z".join( fields[:2] ).lower()
+ if zone in zones:
+ zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
+ elif fields[6]:
+ state = fields[0]
+ description = fields[3]
+ county = fields[5]
+ fips = "fips%s"%fields[6]
+ possible = [
+ "%s, %s" % (county, state),
+ "%s County, %s" % (county, state),
+ ]
+ if description.endswith(" Counties"):
+ description = description[:-9]
+ for addition in description.split(" and "):
+ possible.append( "%s, %s" % (addition, state) )
+ possible.append( "%s County, %s" % (addition, state) )
+ if fips in places and "centroid" in places[fips]:
+ for candidate in zones:
+ if "centroid" not in zones[candidate] and \
+ "description" in zones[candidate] and \
+ zones[candidate]["description"] in possible:
+ zones[candidate]["centroid"] = \
+ places[fips]["centroid"]
+ count += 1
+ cpfzcf.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU")
+ columns = gzcta.readline().decode("latin1").strip().split("\t")
+ for line in gzcta:
+ fields = line.decode("latin1").strip().split("\t")
+ f_geoid = fields[ columns.index("GEOID") ].strip()
+ f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
+ f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
+ if f_geoid and f_intptlat and f_intptlong:
+ if f_geoid not in zctas: zctas[f_geoid] = {}
+ zctas[f_geoid]["centroid"] = gecos(
+ "%s,%s" % (f_intptlat, f_intptlong)
+ )
+ count += 1
+ gzcta.close()
+ print("done (%s lines)." % count)
+ message = "Reading %s..." % overrides_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ added = 0
+ removed = 0
+ changed = 0
+ overrides = configparser.ConfigParser()
+ overrides.readfp( codecs.open(overrides_fn, "r", "utf8") )
+ overrideslog = []
+ for section in overrides.sections():
+ addopt = 0
+ chgopt = 0
+ if section.startswith("-"):
+ section = section[1:]
+ delete = True
+ else: delete = False
+ if re.match("[A-Za-z]{3}$", section):
+ if delete:
+ if section in airports:
+ del( airports[section] )
+ logact = "removed airport %s" % section
+ removed += 1
+ else:
+ logact = "tried to remove nonexistent airport %s" % section
+ else:
+ if section in airports:
+ logact = "changed airport %s" % section
+ changed += 1
+ else:
+ airports[section] = {}
+ logact = "added airport %s" % section
+ added += 1
+ for key,value in overrides.items(section):
+ if key in airports[section]: chgopt += 1
+ else: addopt += 1
+ if key in ("centroid", "location"):
+ airports[section][key] = eval(value)
+ else:
+ airports[section][key] = value
+ if addopt and chgopt:
+ logact += " (+%s/!%s options)" % (addopt, chgopt)
+ elif addopt: logact += " (+%s options)" % addopt
+ elif chgopt: logact += " (!%s options)" % chgopt
+ elif re.match("[A-Za-z0-9]{4}$", section):
+ if delete:
+ if section in stations:
+ del( stations[section] )
+ logact = "removed station %s" % section
+ removed += 1
+ else:
+ logact = "tried to remove nonexistent station %s" % section
+ else:
+ if section in stations:
+ logact = "changed station %s" % section
+ changed += 1
+ else:
+ stations[section] = {}
+ logact = "added station %s" % section
+ added += 1
+ for key,value in overrides.items(section):
+ if key in stations[section]: chgopt += 1
+ else: addopt += 1
+ if key in ("centroid", "location"):
+ stations[section][key] = eval(value)
+ else:
+ stations[section][key] = value
+ if addopt and chgopt:
+ logact += " (+%s/!%s options)" % (addopt, chgopt)
+ elif addopt: logact += " (+%s options)" % addopt
+ elif chgopt: logact += " (!%s options)" % chgopt
+ elif re.match("[0-9]{5}$", section):
+ if delete:
+ if section in zctas:
+ del( zctas[section] )
+ logact = "removed zcta %s" % section
+ removed += 1
+ else:
+ logact = "tried to remove nonexistent zcta %s" % section
+ else:
+ if section in zctas:
+ logact = "changed zcta %s" % section
+ changed += 1
+ else:
+ zctas[section] = {}
+ logact = "added zcta %s" % section
+ added += 1
+ for key,value in overrides.items(section):
+ if key in zctas[section]: chgopt += 1
+ else: addopt += 1
+ if key in ("centroid", "location"):
+ zctas[section][key] = eval(value)
+ else:
+ zctas[section][key] = value
+ if addopt and chgopt:
+ logact += " (+%s/!%s options)" % (addopt, chgopt)
+ elif addopt: logact += " (+%s options)" % addopt
+ elif chgopt: logact += " (!%s options)" % chgopt
+ elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
+ if delete:
+ if section in zones:
+ del( zones[section] )
+ logact = "removed zone %s" % section
+ removed += 1
+ else:
+ logact = "tried to remove nonexistent zone %s" % section
+ else:
+ if section in zones:
+ logact = "changed zone %s" % section
+ changed += 1
+ else:
+ zones[section] = {}
+ logact = "added zone %s" % section
+ added += 1
+ for key,value in overrides.items(section):
+ if key in zones[section]: chgopt += 1
+ else: addopt += 1
+ if key in ("centroid", "location"):
+ zones[section][key] = eval(value)
+ else:
+ zones[section][key] = value
+ if addopt and chgopt:
+ logact += " (+%s/!%s options)" % (addopt, chgopt)
+ elif addopt: logact += " (+%s options)" % addopt
+ elif chgopt: logact += " (!%s options)" % chgopt
+ elif re.match("fips[0-9]+$", section):
+ if delete:
+ if section in places:
+ del( places[section] )
+ logact = "removed place %s" % section
+ removed += 1
+ else:
+ logact = "tried to remove nonexistent place %s" % section
+ else:
+ if section in places:
+ logact = "changed place %s" % section
+ changed += 1
+ else:
+ places[section] = {}
+ logact = "added place %s" % section
+ added += 1
+ for key,value in overrides.items(section):
+ if key in places[section]: chgopt += 1
+ else: addopt += 1
+ if key in ("centroid", "location"):
+ places[section][key] = eval(value)
+ else:
+ places[section][key] = value
+ if addopt and chgopt:
+ logact += " (+%s/!%s options)" % (addopt, chgopt)
+ elif addopt: logact += " (+%s options)" % addopt
+ elif chgopt: logact += " (!%s options)" % chgopt
+ count += 1
+ overrideslog.append("%s\n" % logact)
+ overrideslog.sort()
+ if os.path.exists(overrideslog_fn):
+ os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
+ overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
+ overrideslog_fd.writelines(overrideslog)
+ overrideslog_fd.close()
+ print("done (%s overridden sections: +%s/-%s/!%s)." % (
+ count,
+ added,
+ removed,
+ changed
+ ) )
+ estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
+ print(
+ "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
+ estimate
+ )
+ count = 0
+ milestones = list( range(51) )
+ message = " "
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ for fips in places:
+ centroid = places[fips]["centroid"]
+ if centroid:
+ station = closest(centroid, stations, "location", 0.1)
+ if station[0]:
+ places[fips]["station"] = station
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[:milestones.index(level)+1]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ if centroid:
+ zone = closest(centroid, zones, "centroid", 0.1)
+ if zone[0]:
+ places[fips]["zone"] = zone
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[:milestones.index(level)+1]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ for station in stations:
+ if "location" in stations[station]:
+ location = stations[station]["location"]
+ if location:
+ zone = closest(location, zones, "centroid", 0.1)
+ if zone[0]:
+ stations[station]["zone"] = zone
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[:milestones.index(level)+1]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ for zcta in zctas.keys():
+ centroid = zctas[zcta]["centroid"]
+ if centroid:
+ station = closest(centroid, stations, "location", 0.1)
+ if station[0]:
+ zctas[zcta]["station"] = station
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[ : milestones.index(level)+1 ]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ if centroid:
+ zone = closest(centroid, zones, "centroid", 0.1)
+ if zone[0]:
+ zctas[zcta]["zone"] = zone
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[:milestones.index(level)+1]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ for zone in zones.keys():
+ if "centroid" in zones[zone]:
+ centroid = zones[zone]["centroid"]
+ if centroid:
+ station = closest(centroid, stations, "location", 0.1)
+ if station[0]:
+ zones[zone]["station"] = station
+ count += 1
+ if not count%100:
+ level = int(50*count/estimate)
+ if level in milestones:
+ for remaining in milestones[:milestones.index(level)+1]:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ milestones.remove(remaining)
+ for remaining in milestones:
+ if remaining%5:
+ message = "."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ else:
+ message = "%s%%" % (remaining*2,)
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ print("\n done (%s correlations)." % count)
+ message = "Writing %s..." % airports_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ if os.path.exists(airports_fn):
+ os.rename(airports_fn, "%s_old"%airports_fn)
+ airports_fd = codecs.open(airports_fn, "w", "utf8")
+ airports_fd.write(header)
+ for airport in sorted( airports.keys() ):
+ airports_fd.write("\n\n[%s]" % airport)
+ for key, value in sorted( airports[airport].items() ):
+ if type(value) is float: value = "%.7f"%value
+ elif type(value) is tuple:
+ elements = []
+ for element in value:
+ if type(element) is float: elements.append("%.7f"%element)
+ else: elements.append( repr(element) )
+ value = "(%s)"%", ".join(elements)
+ airports_fd.write( "\n%s = %s" % (key, value) )
+ count += 1
+ airports_fd.write("\n")
+ airports_fd.close()
+ print("done (%s sections)." % count)
+ message = "Writing %s..." % places_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ if os.path.exists(places_fn):
+ os.rename(places_fn, "%s_old"%places_fn)
+ places_fd = codecs.open(places_fn, "w", "utf8")
+ places_fd.write(header)
+ for fips in sorted( places.keys() ):
+ places_fd.write("\n\n[%s]" % fips)
+ for key, value in sorted( places[fips].items() ):
+ if type(value) is float: value = "%.7f"%value
+ elif type(value) is tuple:
+ elements = []
+ for element in value:
+ if type(element) is float: elements.append("%.7f"%element)
+ else: elements.append( repr(element) )
+ value = "(%s)"%", ".join(elements)
+ places_fd.write( "\n%s = %s" % (key, value) )
+ count += 1
+ places_fd.write("\n")
+ places_fd.close()
+ print("done (%s sections)." % count)
+ message = "Writing %s..." % stations_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ if os.path.exists(stations_fn):
+ os.rename(stations_fn, "%s_old"%stations_fn)
+ stations_fd = codecs.open(stations_fn, "w", "utf8")
+ stations_fd.write(header)
+ for station in sorted( stations.keys() ):
+ stations_fd.write("\n\n[%s]" % station)
+ for key, value in sorted( stations[station].items() ):
+ if type(value) is float: value = "%.7f"%value
+ elif type(value) is tuple:
+ elements = []
+ for element in value:
+ if type(element) is float: elements.append("%.7f"%element)
+ else: elements.append( repr(element) )
+ value = "(%s)"%", ".join(elements)
+ stations_fd.write( "\n%s = %s" % (key, value) )
+ count += 1
+ stations_fd.write("\n")
+ stations_fd.close()
+ print("done (%s sections)." % count)
+ message = "Writing %s..." % zctas_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ if os.path.exists(zctas_fn):
+ os.rename(zctas_fn, "%s_old"%zctas_fn)
+ zctas_fd = codecs.open(zctas_fn, "w", "utf8")
+ zctas_fd.write(header)
+ for zcta in sorted( zctas.keys() ):
+ zctas_fd.write("\n\n[%s]" % zcta)
+ for key, value in sorted( zctas[zcta].items() ):
+ if type(value) is float: value = "%.7f"%value
+ elif type(value) is tuple:
+ elements = []
+ for element in value:
+ if type(element) is float: elements.append("%.7f"%element)
+ else: elements.append( repr(element) )
+ value = "(%s)"%", ".join(elements)
+ zctas_fd.write( "\n%s = %s" % (key, value) )
+ count += 1
+ zctas_fd.write("\n")
+ zctas_fd.close()
+ print("done (%s sections)." % count)
+ message = "Writing %s..." % zones_fn
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ count = 0
+ if os.path.exists(zones_fn):
+ os.rename(zones_fn, "%s_old"%zones_fn)
+ zones_fd = codecs.open(zones_fn, "w", "utf8")
+ zones_fd.write(header)
+ for zone in sorted( zones.keys() ):
+ zones_fd.write("\n\n[%s]" % zone)
+ for key, value in sorted( zones[zone].items() ):
+ if type(value) is float: value = "%.7f"%value
+ elif type(value) is tuple:
+ elements = []
+ for element in value:
+ if type(element) is float: elements.append("%.7f"%element)
+ else: elements.append( repr(element) )
+ value = "(%s)"%", ".join(elements)
+ zones_fd.write( "\n%s = %s" % (key, value) )
+ count += 1
+ zones_fd.write("\n")
+ zones_fd.close()
+ print("done (%s sections)." % count)
+ message = "Starting QA check..."
+ sys.stdout.write(message)
+ sys.stdout.flush()
+ airports = configparser.ConfigParser()
+ airports.read(airports_fn)
+ places = configparser.ConfigParser()
+ places.read(places_fn)
+ stations = configparser.ConfigParser()
+ stations.read(stations_fn)
+ zctas = configparser.ConfigParser()
+ zctas.read(zctas_fn)
+ zones = configparser.ConfigParser()
+ zones.read(zones_fn)
+ qalog = []
+ places_nocentroid = 0
+ places_nodescription = 0
+ for place in sorted( places.sections() ):
+ if not places.has_option(place, "centroid"):
+ qalog.append("%s: no centroid\n" % place)
+ places_nocentroid += 1
+ if not places.has_option(place, "description"):
+ qalog.append("%s: no description\n" % place)
+ places_nodescription += 1
+ stations_nodescription = 0
+ stations_nolocation = 0
+ stations_nometar = 0
+ for station in sorted( stations.sections() ):
+ if not stations.has_option(station, "description"):
+ qalog.append("%s: no description\n" % station)
+ stations_nodescription += 1
+ if not stations.has_option(station, "location"):
+ qalog.append("%s: no location\n" % station)
+ stations_nolocation += 1
+ if not stations.has_option(station, "metar"):
+ qalog.append("%s: no metar\n" % station)
+ stations_nometar += 1
+ airports_badstation = 0
+ airports_nostation = 0
+ for airport in sorted( airports.sections() ):
+ if not airports.has_option(airport, "station"):
+ qalog.append("%s: no station\n" % airport)
+ airports_nostation += 1
+ else:
+ station = airports.get(airport, "station")
+ if station not in stations.sections():
+ qalog.append( "%s: bad station %s\n" % (airport, station) )
+ airports_badstation += 1
+ zctas_nocentroid = 0
+ for zcta in sorted( zctas.sections() ):
+ if not zctas.has_option(zcta, "centroid"):
+ qalog.append("%s: no centroid\n" % zcta)
+ zctas_nocentroid += 1
+ zones_nocentroid = 0
+ zones_nodescription = 0
+ zones_noforecast = 0
+ zones_overlapping = 0
+ zonetable = {}
+ for zone in zones.sections():
+ if zones.has_option(zone, "centroid"):
+ zonetable[zone] = {
+ "centroid": eval( zones.get(zone, "centroid") )
+ }
+ for zone in sorted( zones.sections() ):
+ if zones.has_option(zone, "centroid"):
+ zonetable_local = zonetable.copy()
+ del( zonetable_local[zone] )
+ centroid = eval( zones.get(zone, "centroid") )
+ if centroid:
+ nearest = closest(centroid, zonetable_local, "centroid", 0.1)
+ if nearest[1]*radian_to_km < 1:
+ qalog.append( "%s: within one km of %s\n" % (
+ zone,
+ nearest[0]
+ ) )
+ zones_overlapping += 1
+ else:
+ qalog.append("%s: no centroid\n" % zone)
+ zones_nocentroid += 1
+ if not zones.has_option(zone, "description"):
+ qalog.append("%s: no description\n" % zone)
+ zones_nodescription += 1
+ if not zones.has_option(zone, "zone_forecast"):
+ qalog.append("%s: no forecast\n" % zone)
+ zones_noforecast += 1
+ if os.path.exists(qalog_fn):
+ os.rename(qalog_fn, "%s_old"%qalog_fn)
+ qalog_fd = codecs.open(qalog_fn, "w", "utf8")
+ qalog_fd.writelines(qalog)
+ qalog_fd.close()
+ if qalog:
+ print("issues found (see %s for details):"%qalog_fn)
+ if airports_badstation:
+ print(" %s airports with invalid station"%airports_badstation)
+ if airports_nostation:
+ print(" %s airports with no station"%airports_nostation)
+ if places_nocentroid:
+ print(" %s places with no centroid"%places_nocentroid)
+ if places_nodescription:
+ print(" %s places with no description"%places_nodescription)
+ if stations_nodescription:
+ print(" %s stations with no description"%stations_nodescription)
+ if stations_nolocation:
+ print(" %s stations with no location"%stations_nolocation)
+ if stations_nometar:
+ print(" %s stations with no METAR"%stations_nometar)
+ if zctas_nocentroid:
+ print(" %s ZCTAs with no centroid"%zctas_nocentroid)
+ if zones_nocentroid:
+ print(" %s zones with no centroid"%zones_nocentroid)
+ if zones_nodescription:
+ print(" %s zones with no description"%zones_nodescription)
+ if zones_noforecast:
+ print(" %s zones with no forecast"%zones_noforecast)
+ if zones_overlapping:
+ print(" %s zones within one km of another"%zones_overlapping)
+ else: print("no issues found.")
+ print("Indexing complete!")