weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2024 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.4.4"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 def pyversion(ref=None):
  15     """Determine the Python version and optionally compare to a reference."""
  16     import platform
  17     ver = platform.python_version()
  18     if ref:
  19         return [
  20             int(x) for x in ver.split(".")[:2]
  21         ] >= [
  22             int(x) for x in ref.split(".")[:2]
  23         ]
  24     else: return ver
  25
  26 class Selections:
  27     """An object to contain selection data."""
  28     def __init__(self):
  29         """Store the config, options and arguments."""
  30         self.config = get_config()
  31         self.options, self.arguments = get_options(self.config)
  32         if self.get_bool("cache") and self.get_bool("cache_search") \
  33             and not self.get_bool("longlist"):
  34             integrate_search_cache(
  35                 self.config,
  36                 self.get("cachedir"),
  37                 self.get("setpath")
  38             )
  39         if not self.arguments:
  40             if "id" in self.options.__dict__ \
  41                 and self.options.__dict__["id"]:
  42                 self.arguments.append( self.options.__dict__["id"] )
  43                 del( self.options.__dict__["id"] )
  44                 import sys
  45                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  46                 sys.stderr.write(message)
  47             elif "city" in self.options.__dict__ \
  48                 and self.options.__dict__["city"] \
  49                 and "st" in self.options.__dict__ \
  50                 and self.options.__dict__["st"]:
  51                 self.arguments.append(
  52                     "^%s city, %s" % (
  53                         self.options.__dict__["city"],
  54                         self.options.__dict__["st"]
  55                     )
  56                 )
  57                 del( self.options.__dict__["city"] )
  58                 del( self.options.__dict__["st"] )
  59                 import sys
  60                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  61                 sys.stderr.write(message)
  62     def get(self, option, argument=None):
  63         """Retrieve data from the config or options."""
  64         if argument:
  65             if self.config.has_section(argument) and (
  66                 self.config.has_option(argument, "city") \
  67                     or self.config.has_option(argument, "id") \
  68                     or self.config.has_option(argument, "st")
  69             ):
  70                 self.config.remove_section(argument)
  71                 import sys
  72                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  73                 sys.stderr.write(message)
  74             if not self.config.has_section(argument):
  75                 guessed = guess(
  76                     argument,
  77                     path=self.get("setpath"),
  78                     info=self.get("info"),
  79                     cache_search=(
  80                         self.get("cache") and self.get("cache_search")
  81                     ),
  82                     cachedir=self.get("cachedir"),
  83                     quiet=self.get_bool("quiet")
  84                 )
  85                 self.config.add_section(argument)
  86                 for item in guessed.items():
  87                     self.config.set(argument, *item)
  88             if self.config.has_option(argument, option):
  89                 return self.config.get(argument, option)
  90         if option in self.options.__dict__:
  91             return self.options.__dict__[option]
  92         import sys
  93         message = "WARNING: no URI defined for %s\n" % option
  94         sys.stderr.write(message)
  95         return None
  96     def get_bool(self, option, argument=None):
  97         """Get data and coerce to a boolean if necessary."""
  98         # Mimic configparser's getboolean() method by treating
  99         # false/no/off/0 as False and true/yes/on/1 as True values,
 100         # case-insensitively
 101         value = self.get(option, argument)
 102         if isinstance(value, bool):
 103             return value
 104         if isinstance(value, str):
 105             vlower = value.lower()
 106             if vlower in ('false', 'no', 'off', '0'):
 107                 return False
 108             elif vlower in ('true', 'yes', 'on', '1'):
 109                 return True
 110         raise ValueError("Not a boolean: %s" % value)
 111     def getint(self, option, argument=None):
 112         """Get data and coerce to an integer if necessary."""
 113         value = self.get(option, argument)
 114         if value: return int(value)
 115         else: return 0
 116
 117 def average(coords):
 118     """Average a list of coordinates."""
 119     x = 0
 120     y = 0
 121     for coord in coords:
 122         x += coord[0]
 123         y += coord[1]
 124     count = len(coords)
 125     return (x/count, y/count)
 126
 127 def filter_units(line, units="imperial"):
 128     """Filter or convert units in a line of text between US/UK and metric."""
 129     import re
 130     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 131     # "X in. Hg (Y hPa)"
 132     dual_p = re.match(
 133         r"(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 134         line
 135     )
 136     if dual_p:
 137         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 138         if units == "imperial": line = preamble + in_hg + trailer
 139         elif units == "metric": line = preamble + hpa + trailer
 140     # filter lines with both temperatures in the form of "X F (Y C)"
 141     dual_t = re.match(
 142         r"(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 143         line
 144     )
 145     if dual_t:
 146         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 147         if units == "imperial": line = preamble + fahrenheit + trailer
 148         elif units == "metric": line = preamble + celsius + trailer
 149     # if metric is desired, convert distances in the form of "X mile(s)" to
 150     # "Y kilometer(s)"
 151     if units == "metric":
 152         imperial_d = re.match(
 153             r"(.* )(\d+)( mile\(s\))(.*)",
 154             line
 155         )
 156         if imperial_d:
 157             preamble, mi, m_u, trailer = imperial_d.groups()
 158             line = preamble + str(int(round(int(mi)*1.609344))) \
 159                 + " kilometer(s)" + trailer
 160     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 161     # desired, convert to "Z KPH"
 162     imperial_s = re.match(
 163         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 164         line
 165     )
 166     if imperial_s:
 167         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 168         if units == "imperial": line = preamble + mph + m_u + trailer
 169         elif units == "metric":
 170             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 171                 trailer
 172     imperial_s = re.match(
 173         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 174         line
 175     )
 176     if imperial_s:
 177         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 178         if units == "imperial": line = preamble + mph + m_u + trailer
 179         elif units == "metric":
 180             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 181                 trailer
 182     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 183     # metric is desired, convert to "Y C"
 184     imperial_t = re.match(
 185         r"(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 186         line
 187     )
 188     if imperial_t:
 189         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 190         if units == "imperial":
 191             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 192         elif units == "metric":
 193             line = preamble + parameter \
 194                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 195                 + trailer
 196     # hand off the resulting line
 197     return line
 198
 199 def get_uri(
 200     uri,
 201     ignore_fail=False,
 202     cache_data=False,
 203     cacheage=900,
 204     cachedir="."
 205 ):
 206     """Return a string containing the results of a URI GET."""
 207     if pyversion("3"):
 208         import urllib, urllib.error, urllib.request
 209         URLError = urllib.error.URLError
 210         urlopen = urllib.request.urlopen
 211     else:
 212         import urllib2 as urllib
 213         URLError = urllib.URLError
 214         urlopen = urllib.urlopen
 215     import os, time
 216     if cache_data:
 217         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 218         if not os.path.exists(dcachedir):
 219             try: os.makedirs(dcachedir)
 220             except (IOError, OSError): pass
 221         dcache_fn = os.path.join(
 222             dcachedir,
 223             uri.split(":",1)[1].replace("/","_")
 224         )
 225     now = time.time()
 226     if cache_data and os.access(dcache_fn, os.R_OK) \
 227         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 228         dcache_fd = open(dcache_fn)
 229         data = dcache_fd.read()
 230         dcache_fd.close()
 231     else:
 232         try:
 233             data = urlopen(uri).read().decode("utf-8")
 234         except URLError:
 235             if ignore_fail: return ""
 236             import os, sys
 237             sys.stderr.write("%s error: failed to retrieve\n   %s\n\n" % (
 238                 os.path.basename( sys.argv[0] ), uri))
 239             raise
 240         # Some data sources are HTML with the plain text wrapped in pre tags
 241         if "<pre>" in data:
 242             data = data[data.find("<pre>")+5:data.find("</pre>")]
 243         if cache_data:
 244             try:
 245                 import codecs
 246                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 247                 dcache_fd.write(data)
 248                 dcache_fd.close()
 249             except (IOError, OSError): pass
 250     return data
 251
 252 def get_metar(
 253     uri=None,
 254     verbose=False,
 255     quiet=False,
 256     headers=None,
 257     imperial=False,
 258     metric=False,
 259     cache_data=False,
 260     cacheage=900,
 261     cachedir="."
 262 ):
 263     """Return a summarized METAR for the specified station."""
 264     if not uri:
 265         import os, sys
 266         message = "%s error: METAR URI required for conditions\n" % \
 267             os.path.basename( sys.argv[0] )
 268         sys.stderr.write(message)
 269         sys.exit(1)
 270     metar = get_uri(
 271         uri,
 272         cache_data=cache_data,
 273         cacheage=cacheage,
 274         cachedir=cachedir
 275     )
 276     if pyversion("3") and type(metar) is bytes: metar = metar.decode("utf-8")
 277     if verbose: return metar
 278     else:
 279         import re
 280         lines = metar.split("\n")
 281         if not headers:
 282             headers = \
 283                 "relative_humidity," \
 284                 + "precipitation_last_hour," \
 285                 + "sky conditions," \
 286                 + "temperature," \
 287                 + "heat index," \
 288                 + "windchill," \
 289                 + "weather," \
 290                 + "wind"
 291         headerlist = headers.lower().replace("_"," ").split(",")
 292         output = []
 293         if not quiet:
 294             title = "Current conditions at %s"
 295             place = lines[0].split(", ")
 296             if len(place) > 1:
 297                 place = "%s, %s" % ( place[0].title(), place[1] )
 298             else: place = "<UNKNOWN>"
 299             output.append(title%place)
 300             output.append("Last updated " + lines[1])
 301         header_match = False
 302         for header in headerlist:
 303             for line in lines:
 304                 if line.lower().startswith(header + ":"):
 305                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 306                     if imperial: line = filter_units(line, units="imperial")
 307                     elif metric: line = filter_units(line, units="metric")
 308                     if quiet: output.append(line)
 309                     else: output.append("   " + line)
 310                     header_match = True
 311         if not header_match:
 312             output.append(
 313                 "(no conditions matched your header list, try with --verbose)"
 314             )
 315         return "\n".join(output)
 316
 317 def get_alert(
 318     uri=None,
 319     verbose=False,
 320     quiet=False,
 321     cache_data=False,
 322     cacheage=900,
 323     cachedir="."
 324 ):
 325     """Return alert notice for the specified URI."""
 326     if not uri:
 327         return ""
 328     alert = get_uri(
 329         uri,
 330         ignore_fail=True,
 331         cache_data=cache_data,
 332         cacheage=cacheage,
 333         cachedir=cachedir
 334     ).strip()
 335     if pyversion("3") and type(alert) is bytes: alert = alert.decode("utf-8")
 336     if alert:
 337         if verbose: return alert
 338         else:
 339             if alert.find("\nNATIONAL WEATHER SERVICE") == -1:
 340                 muted = False
 341             else:
 342                 muted = True
 343             lines = alert.split("\n")
 344             import time
 345             # TODO: make this offset configurable
 346             # TODO: adjust offset relative to the difference between the user's
 347             #       local time and the zone's local time (will need to extend
 348             #       the schema in the zones file to store each tz
 349             offset = 86400  # one day
 350
 351             # report alerts and forecasts that expired less than offset ago;
 352             # this is a cheap hack since expiration times seem to be relative
 353             # to the zone's local time zone, and converting from the user's
 354             # would get complicated, but also there can sometimes be a lag
 355             # between expiration and the next update
 356             valid_time = time.strftime(
 357                 "%Y%m%d%H%M", time.localtime(time.time() - offset))
 358             output = []
 359             for line in lines:
 360                 if line.startswith("Expires:") \
 361                     and "Expires:" + valid_time > line:
 362                     return ""
 363                 if muted and line.startswith("NATIONAL WEATHER SERVICE"):
 364                     muted = False
 365                     line = ""
 366                 elif line == "&&":
 367                     line = ""
 368                 elif line == "$$":
 369                     muted = True
 370                 if line and not muted:
 371                     if quiet: output.append(line)
 372                     else: output.append("   " + line)
 373             return "\n".join(output)
 374
 375 def get_options(config):
 376     """Parse the options passed on the command line."""
 377
 378     # for optparse's builtin -h/--help option
 379     usage = \
 380         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 381
 382     # for optparse's builtin --version option
 383     verstring = "%prog " + weather_version
 384
 385     # create the parser
 386     import optparse
 387     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 388     # separate options object from list of arguments and return both
 389
 390     # the -a/--alert option
 391     if config.has_option("default", "alert"):
 392         default_alert = config.getboolean("default", "alert")
 393     else: default_alert = False
 394     option_parser.add_option("-a", "--alert",
 395         dest="alert",
 396         action="store_true",
 397         default=default_alert,
 398         help="include local alert notices")
 399
 400     # the --atypes option
 401     if config.has_option("default", "atypes"):
 402         default_atypes = config.get("default", "atypes")
 403     else:
 404         default_atypes = \
 405             "coastal_flood_statement," \
 406             + "flash_flood_statement," \
 407             + "flash_flood_warning," \
 408             + "flash_flood_watch," \
 409             + "flood_warning," \
 410             + "severe_thunderstorm_warning," \
 411             + "severe_weather_statement," \
 412             + "special_weather_statement," \
 413             + "tornado," \
 414             + "urgent_weather_message"
 415     option_parser.add_option("--atypes",
 416         dest="atypes",
 417         default=default_atypes,
 418         help="list of alert notification types to display")
 419
 420     # the --build-sets option
 421     option_parser.add_option("--build-sets",
 422         dest="build_sets",
 423         action="store_true",
 424         default=False,
 425         help="(re)build location correlation sets")
 426
 427     # the --cacheage option
 428     if config.has_option("default", "cacheage"):
 429         default_cacheage = config.getint("default", "cacheage")
 430     else: default_cacheage = 900
 431     option_parser.add_option("--cacheage",
 432         dest="cacheage",
 433         default=default_cacheage,
 434         help="duration in seconds to refresh cached data")
 435
 436     # the --cachedir option
 437     if config.has_option("default", "cachedir"):
 438         default_cachedir = config.get("default", "cachedir")
 439     else: default_cachedir = "~/.weather"
 440     option_parser.add_option("--cachedir",
 441         dest="cachedir",
 442         default=default_cachedir,
 443         help="directory for storing cached searches and data")
 444
 445     # the -f/--forecast option
 446     if config.has_option("default", "forecast"):
 447         default_forecast = config.getboolean("default", "forecast")
 448     else: default_forecast = False
 449     option_parser.add_option("-f", "--forecast",
 450         dest="forecast",
 451         action="store_true",
 452         default=default_forecast,
 453         help="include a local forecast")
 454
 455     # the --headers option
 456     if config.has_option("default", "headers"):
 457         default_headers = config.get("default", "headers")
 458     else:
 459         default_headers = \
 460             "temperature," \
 461             + "relative_humidity," \
 462             + "wind," \
 463             + "heat_index," \
 464             + "windchill," \
 465             + "weather," \
 466             + "sky_conditions," \
 467             + "precipitation_last_hour"
 468     option_parser.add_option("--headers",
 469         dest="headers",
 470         default=default_headers,
 471         help="list of conditions headers to display")
 472
 473     # the --imperial option
 474     if config.has_option("default", "imperial"):
 475         default_imperial = config.getboolean("default", "imperial")
 476     else: default_imperial = False
 477     option_parser.add_option("--imperial",
 478         dest="imperial",
 479         action="store_true",
 480         default=default_imperial,
 481         help="filter/convert conditions for US/UK units")
 482
 483     # the --info option
 484     option_parser.add_option("--info",
 485         dest="info",
 486         action="store_true",
 487         default=False,
 488         help="output detailed information for your search")
 489
 490     # the -l/--list option
 491     option_parser.add_option("-l", "--list",
 492         dest="list",
 493         action="store_true",
 494         default=False,
 495         help="list all configured aliases and cached searches")
 496
 497     # the --longlist option
 498     option_parser.add_option("--longlist",
 499         dest="longlist",
 500         action="store_true",
 501         default=False,
 502         help="display details of all configured aliases")
 503
 504     # the -m/--metric option
 505     if config.has_option("default", "metric"):
 506         default_metric = config.getboolean("default", "metric")
 507     else: default_metric = False
 508     option_parser.add_option("-m", "--metric",
 509         dest="metric",
 510         action="store_true",
 511         default=default_metric,
 512         help="filter/convert conditions for metric units")
 513
 514     # the -n/--no-conditions option
 515     if config.has_option("default", "conditions"):
 516         default_conditions = config.getboolean("default", "conditions")
 517     else: default_conditions = True
 518     option_parser.add_option("-n", "--no-conditions",
 519         dest="conditions",
 520         action="store_false",
 521         default=default_conditions,
 522         help="disable output of current conditions")
 523
 524     # the --no-cache option
 525     if config.has_option("default", "cache"):
 526         default_cache = config.getboolean("default", "cache")
 527     else: default_cache = True
 528     option_parser.add_option("--no-cache",
 529         dest="cache",
 530         action="store_false",
 531         default=True,
 532         help="disable all caching (searches and data)")
 533
 534     # the --no-cache-data option
 535     if config.has_option("default", "cache_data"):
 536         default_cache_data = config.getboolean("default", "cache_data")
 537     else: default_cache_data = True
 538     option_parser.add_option("--no-cache-data",
 539         dest="cache_data",
 540         action="store_false",
 541         default=True,
 542         help="disable retrieved data caching")
 543
 544     # the --no-cache-search option
 545     if config.has_option("default", "cache_search"):
 546         default_cache_search = config.getboolean("default", "cache_search")
 547     else: default_cache_search = True
 548     option_parser.add_option("--no-cache-search",
 549         dest="cache_search",
 550         action="store_false",
 551         default=True,
 552         help="disable search result caching")
 553
 554     # the -q/--quiet option
 555     if config.has_option("default", "quiet"):
 556         default_quiet = config.getboolean("default", "quiet")
 557     else: default_quiet = False
 558     option_parser.add_option("-q", "--quiet",
 559         dest="quiet",
 560         action="store_true",
 561         default=default_quiet,
 562         help="skip preambles and don't indent")
 563
 564     # the --setpath option
 565     if config.has_option("default", "setpath"):
 566         default_setpath = config.get("default", "setpath")
 567     else: default_setpath = ".:~/.weather"
 568     option_parser.add_option("--setpath",
 569         dest="setpath",
 570         default=default_setpath,
 571         help="directory search path for correlation sets")
 572
 573     # the -v/--verbose option
 574     if config.has_option("default", "verbose"):
 575         default_verbose = config.getboolean("default", "verbose")
 576     else: default_verbose = False
 577     option_parser.add_option("-v", "--verbose",
 578         dest="verbose",
 579         action="store_true",
 580         default=default_verbose,
 581         help="show full decoded feeds")
 582
 583     # deprecated options
 584     if config.has_option("default", "city"):
 585         default_city = config.get("default", "city")
 586     else: default_city = ""
 587     option_parser.add_option("-c", "--city",
 588         dest="city",
 589         default=default_city,
 590         help=optparse.SUPPRESS_HELP)
 591     if config.has_option("default", "id"):
 592         default_id = config.get("default", "id")
 593     else: default_id = ""
 594     option_parser.add_option("-i", "--id",
 595         dest="id",
 596         default=default_id,
 597         help=optparse.SUPPRESS_HELP)
 598     if config.has_option("default", "st"):
 599         default_st = config.get("default", "st")
 600     else: default_st = ""
 601     option_parser.add_option("-s", "--st",
 602         dest="st",
 603         default=default_st,
 604         help=optparse.SUPPRESS_HELP)
 605
 606     options, arguments = option_parser.parse_args()
 607     return options, arguments
 608
 609 def get_config():
 610     """Parse the aliases and configuration."""
 611     if pyversion("3"): import configparser
 612     else: import ConfigParser as configparser
 613     config = configparser.ConfigParser()
 614     import os
 615     rcfiles = [
 616         "/etc/weatherrc",
 617         "/etc/weather/weatherrc",
 618         os.path.expanduser("~/.weather/weatherrc"),
 619         os.path.expanduser("~/.weatherrc"),
 620         "weatherrc"
 621         ]
 622     for rcfile in rcfiles:
 623         if os.access(rcfile, os.R_OK):
 624             if pyversion("3"):
 625                 config.read(rcfile, encoding="utf-8")
 626             else:
 627                 config.read(rcfile)
 628     for section in config.sections():
 629         if section != section.lower():
 630             if config.has_section(section.lower()):
 631                 config.remove_section(section.lower())
 632             config.add_section(section.lower())
 633             for option,value in config.items(section):
 634                 config.set(section.lower(), option, value)
 635     return config
 636
 637 def integrate_search_cache(config, cachedir, setpath):
 638     """Add cached search results into the configuration."""
 639     if pyversion("3"): import configparser
 640     else: import ConfigParser as configparser
 641     import os, time
 642     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 643     if not os.access(scache_fn, os.R_OK): return config
 644     scache_fd = open(scache_fn)
 645     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 646     scache_fd.close()
 647     now = time.time()
 648     datafiles = data_index(setpath)
 649     if datafiles:
 650         data_freshness = sorted(
 651             [ x[1] for x in datafiles.values() ],
 652             reverse=True
 653         )[0]
 654     else: data_freshness = now
 655     if created < data_freshness <= now:
 656         try:
 657             os.remove(scache_fn)
 658             print( "[clearing outdated %s]" % scache_fn )
 659         except (IOError, OSError):
 660             pass
 661         return config
 662     scache = configparser.ConfigParser()
 663     if pyversion("3"):
 664         scache.read(scache_fn, encoding="utf-8")
 665     else:
 666         scache.read(scache_fn)
 667     for section in scache.sections():
 668         if not config.has_section(section):
 669             config.add_section(section)
 670             for option,value in scache.items(section):
 671                 config.set(section, option, value)
 672     return config
 673
 674 def list_aliases(config, detail=False):
 675     """Return a formatted list of aliases defined in the config."""
 676     if detail:
 677         output = "\n# configured alias details..."
 678         for section in sorted(config.sections()):
 679             output += "\n\n[%s]" % section
 680             for item in sorted(config.items(section)):
 681                 output += "\n%s = %s" % item
 682         output += "\n"
 683     else:
 684         output = "configured aliases and cached searches..."
 685         for section in sorted(config.sections()):
 686             if config.has_option(section, "description"):
 687                 description = config.get(section, "description")
 688             else: description = "(no description provided)"
 689             output += "\n   %s: %s" % (section, description)
 690     return output
 691
 692 def data_index(path):
 693     import os
 694     datafiles = {}
 695     for filename in ("airports", "places", "stations", "zctas", "zones"):
 696         for dirname in path.split(":"):
 697             for extension in ("", ".gz", ".txt"):
 698                 candidate = os.path.expanduser(
 699                     os.path.join( dirname, "".join( (filename, extension) ) )
 700                 )
 701                 if os.path.exists(candidate):
 702                     datafiles[filename] = (
 703                         candidate,
 704                         os.stat(candidate).st_mtime
 705                     )
 706                     break
 707             if filename in datafiles:
 708                 break
 709     return datafiles
 710
 711 def guess(
 712     expression,
 713     path=".",
 714     max_results=20,
 715     info=False,
 716     cache_search=False,
 717     cacheage=900,
 718     cachedir=".",
 719     quiet=False
 720 ):
 721     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 722     import codecs, datetime, time, os, re, sys
 723     if pyversion("3"): import configparser
 724     else: import ConfigParser as configparser
 725     datafiles = data_index(path)
 726     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 727     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 728     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 729     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 730     elif re.match(
 731         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 732         expression
 733     ):
 734         searchtype = "coordinates"
 735     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 736     else:
 737         searchtype = "name"
 738         cache_search = False
 739     if cache_search: action = "caching"
 740     else: action = "using"
 741     if info:
 742         scores = [
 743             (0.005, "bad"),
 744             (0.025, "poor"),
 745             (0.160, "suspect"),
 746             (0.500, "mediocre"),
 747             (0.840, "good"),
 748             (0.975, "great"),
 749             (0.995, "excellent"),
 750             (1.000, "ideal"),
 751         ]
 752     if not quiet: print("Searching via %s..."%searchtype)
 753     stations = configparser.ConfigParser()
 754     dataname = "stations"
 755     if dataname in datafiles:
 756         datafile = datafiles[dataname][0]
 757         if datafile.endswith(".gz"):
 758             import gzip
 759             if pyversion("3"):
 760                 stations.read_string(
 761                     gzip.open(datafile).read().decode("utf-8") )
 762             else: stations.read_file( gzip.open(datafile) )
 763         else:
 764             if pyversion("3"):
 765                 stations.read(datafile, encoding="utf-8")
 766             else:
 767                 stations.read(datafile)
 768     else:
 769         message = "%s error: can't find \"%s\" data file\n" % (
 770             os.path.basename( sys.argv[0] ),
 771             dataname
 772         )
 773         sys.stderr.write(message)
 774         exit(1)
 775     zones = configparser.ConfigParser()
 776     dataname = "zones"
 777     if dataname in datafiles:
 778         datafile = datafiles[dataname][0]
 779         if datafile.endswith(".gz"):
 780             import gzip
 781             if pyversion("3"):
 782                 zones.read_string( gzip.open(datafile).read().decode("utf-8") )
 783             else: zones.read_file( gzip.open(datafile) )
 784         else:
 785             if pyversion("3"):
 786                 zones.read(datafile, encoding="utf-8")
 787             else:
 788                 zones.read(datafile)
 789     else:
 790         message = "%s error: can't find \"%s\" data file\n" % (
 791             os.path.basename( sys.argv[0] ),
 792             dataname
 793         )
 794         sys.stderr.write(message)
 795         exit(1)
 796     search = None
 797     station = ("", 0)
 798     zone = ("", 0)
 799     dataset = None
 800     possibilities = []
 801     uris = {}
 802     if searchtype == "airport":
 803         expression = expression.lower()
 804         airports = configparser.ConfigParser()
 805         dataname = "airports"
 806         if dataname in datafiles:
 807             datafile = datafiles[dataname][0]
 808             if datafile.endswith(".gz"):
 809                 import gzip
 810                 if pyversion("3"):
 811                     airports.read_string(
 812                         gzip.open(datafile).read().decode("utf-8") )
 813                 else: airports.read_file( gzip.open(datafile) )
 814             else:
 815                 if pyversion("3"):
 816                     airports.read(datafile, encoding="utf-8")
 817                 else:
 818                     airports.read(datafile)
 819         else:
 820             message = "%s error: can't find \"%s\" data file\n" % (
 821                 os.path.basename( sys.argv[0] ),
 822                 dataname
 823             )
 824             sys.stderr.write(message)
 825             exit(1)
 826         if airports.has_section(expression) \
 827             and airports.has_option(expression, "station"):
 828             search = (expression, "IATA/FAA airport code %s" % expression)
 829             station = ( airports.get(expression, "station"), 0 )
 830             if stations.has_option(station[0], "zone"):
 831                 zone = eval( stations.get(station[0], "zone") )
 832                 dataset = stations
 833             if not ( info or quiet ) \
 834                 and stations.has_option( station[0], "description" ):
 835                 print(
 836                     "[%s result %s]" % (
 837                         action,
 838                         stations.get(station[0], "description")
 839                     )
 840                 )
 841         else:
 842             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 843                 expression,
 844                 datafiles["airports"][0]
 845             )
 846             sys.stderr.write(message)
 847             exit(1)
 848     elif searchtype == "station":
 849         expression = expression.lower()
 850         if stations.has_section(expression):
 851             station = (expression, 0)
 852             if not search:
 853                 search = (expression, "ICAO station code %s" % expression)
 854             if stations.has_option(expression, "zone"):
 855                 zone = eval( stations.get(expression, "zone") )
 856                 dataset = stations
 857             if not ( info or quiet ) \
 858                 and stations.has_option(expression, "description"):
 859                 print(
 860                     "[%s result %s]" % (
 861                         action,
 862                         stations.get(expression, "description")
 863                     )
 864                 )
 865         else:
 866             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 867                 expression,
 868                 datafiles["stations"][0]
 869             )
 870             sys.stderr.write(message)
 871             exit(1)
 872     elif searchtype == "zone":
 873         expression = expression.lower()
 874         if zones.has_section(expression) \
 875             and zones.has_option(expression, "station"):
 876             zone = (expression, 0)
 877             station = eval( zones.get(expression, "station") )
 878             dataset = zones
 879             search = (expression, "NWS/NOAA weather zone %s" % expression)
 880             if not ( info or quiet ) \
 881                 and zones.has_option(expression, "description"):
 882                 print(
 883                     "[%s result %s]" % (
 884                         action,
 885                         zones.get(expression, "description")
 886                     )
 887                 )
 888         else:
 889             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 890                 expression,
 891                 datafiles["zones"][0]
 892             )
 893             sys.stderr.write(message)
 894             exit(1)
 895     elif searchtype == "ZCTA":
 896         zctas = configparser.ConfigParser()
 897         dataname = "zctas"
 898         if dataname in datafiles:
 899             datafile = datafiles[dataname][0]
 900             if datafile.endswith(".gz"):
 901                 import gzip
 902                 if pyversion("3"):
 903                     zctas.read_string(
 904                         gzip.open(datafile).read().decode("utf-8") )
 905                 else: zctas.read_file( gzip.open(datafile) )
 906             else:
 907                 if pyversion("3"):
 908                     zctas.read(datafile, encoding="utf-8")
 909                 else:
 910                     zctas.read(datafile)
 911         else:
 912             message = "%s error: can't find \"%s\" data file\n" % (
 913                 os.path.basename( sys.argv[0] ),
 914                 dataname
 915             )
 916             sys.stderr.write(message)
 917             exit(1)
 918         dataset = zctas
 919         if zctas.has_section(expression) \
 920             and zctas.has_option(expression, "station"):
 921             station = eval( zctas.get(expression, "station") )
 922             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 923             if zctas.has_option(expression, "zone"):
 924                 zone = eval( zctas.get(expression, "zone") )
 925         else:
 926             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 927                 expression,
 928                 datafiles["zctas"][0]
 929             )
 930             sys.stderr.write(message)
 931             exit(1)
 932     elif searchtype == "coordinates":
 933         search = (expression, "Geographic coordinates %s" % expression)
 934         stationtable = {}
 935         for station in stations.sections():
 936             if stations.has_option(station, "location"):
 937                 stationtable[station] = {
 938                     "location": eval( stations.get(station, "location") )
 939                 }
 940         station = closest( gecos(expression), stationtable, "location", 0.1 )
 941         if not station[0]:
 942             message = "No ICAO weather station found near %s.\n" % expression
 943             sys.stderr.write(message)
 944             exit(1)
 945         zonetable = {}
 946         for zone in zones.sections():
 947             if zones.has_option(zone, "centroid"):
 948                 zonetable[zone] = {
 949                     "centroid": eval( zones.get(zone, "centroid") )
 950                 }
 951         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 952         if not zone[0]:
 953             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 954                 % expression
 955             sys.stderr.write(message)
 956     elif searchtype in ("FIPS", "name"):
 957         places = configparser.ConfigParser()
 958         dataname = "places"
 959         if dataname in datafiles:
 960             datafile = datafiles[dataname][0]
 961             if datafile.endswith(".gz"):
 962                 import gzip
 963                 if pyversion("3"):
 964                     places.read_string(
 965                         gzip.open(datafile).read().decode("utf-8") )
 966                 else: places.read_file( gzip.open(datafile) )
 967             else:
 968                 if pyversion("3"):
 969                     places.read(datafile, encoding="utf-8")
 970                 else:
 971                     places.read(datafile)
 972         else:
 973             message = "%s error: can't find \"%s\" data file\n" % (
 974                 os.path.basename( sys.argv[0] ),
 975                 dataname
 976             )
 977             sys.stderr.write(message)
 978             exit(1)
 979         dataset = places
 980         place = expression.lower()
 981         if places.has_section(place) and places.has_option(place, "station"):
 982             station = eval( places.get(place, "station") )
 983             search = (expression, "Census Place %s" % expression)
 984             if places.has_option(place, "description"):
 985                 search = (
 986                     search[0],
 987                     search[1] + ", %s" % places.get(place, "description")
 988                 )
 989             if places.has_option(place, "zone"):
 990                 zone = eval( places.get(place, "zone") )
 991             if not ( info or quiet ) \
 992                 and places.has_option(place, "description"):
 993                 print(
 994                     "[%s result %s]" % (
 995                         action,
 996                         places.get(place, "description")
 997                     )
 998                 )
 999         else:
1000             for place in places.sections():
1001                 if places.has_option(place, "description") \
1002                     and places.has_option(place, "station") \
1003                     and re.search(
1004                         expression,
1005                         places.get(place, "description"),
1006                         re.I
1007                     ):
1008                         possibilities.append(place)
1009             for place in stations.sections():
1010                 if stations.has_option(place, "description") \
1011                     and re.search(
1012                         expression,
1013                         stations.get(place, "description"),
1014                         re.I
1015                     ):
1016                         possibilities.append(place)
1017             for place in zones.sections():
1018                 if zones.has_option(place, "description") \
1019                     and zones.has_option(place, "station") \
1020                     and re.search(
1021                         expression,
1022                         zones.get(place, "description"),
1023                         re.I
1024                     ):
1025                         possibilities.append(place)
1026             if len(possibilities) == 1:
1027                 place = possibilities[0]
1028                 if places.has_section(place):
1029                     station = eval( places.get(place, "station") )
1030                     description = places.get(place, "description")
1031                     if places.has_option(place, "zone"):
1032                         zone = eval( places.get(place, "zone" ) )
1033                     search = ( expression, "%s: %s" % (place, description) )
1034                 elif stations.has_section(place):
1035                     station = (place, 0.0)
1036                     description = stations.get(place, "description")
1037                     if stations.has_option(place, "zone"):
1038                         zone = eval( stations.get(place, "zone" ) )
1039                     search = ( expression, "ICAO station code %s" % place )
1040                 elif zones.has_section(place):
1041                     station = eval( zones.get(place, "station") )
1042                     description = zones.get(place, "description")
1043                     zone = (place, 0.0)
1044                     search = ( expression, "NWS/NOAA weather zone %s" % place )
1045                 if not ( info or quiet ):
1046                     print( "[%s result %s]" % (action, description) )
1047             if not possibilities and not station[0]:
1048                 message = "No FIPS code/census area match in the %s file.\n" % (
1049                     datafiles["places"][0]
1050                 )
1051                 sys.stderr.write(message)
1052                 exit(1)
1053     if station[0]:
1054         uris["metar"] = stations.get( station[0], "metar" )
1055         if zone[0]:
1056             for key,value in zones.items( zone[0] ):
1057                 if key not in ("centroid", "description", "station"):
1058                     uris[key] = value
1059     elif possibilities:
1060         count = len(possibilities)
1061         if count <= max_results:
1062             print( "Your search is ambiguous, returning %s matches:" % count )
1063             for place in sorted(possibilities):
1064                 if places.has_section(place):
1065                     print(
1066                         "   [%s] %s" % (
1067                             place,
1068                             places.get(place, "description")
1069                         )
1070                     )
1071                 elif stations.has_section(place):
1072                     print(
1073                         "   [%s] %s" % (
1074                             place,
1075                             stations.get(place, "description")
1076                         )
1077                     )
1078                 elif zones.has_section(place):
1079                     print(
1080                         "   [%s] %s" % (
1081                             place,
1082                             zones.get(place, "description")
1083                         )
1084                     )
1085         else:
1086             print(
1087                 "Your search is too ambiguous, returning %s matches." % count
1088             )
1089         exit(0)
1090     if info:
1091         stationlist = []
1092         zonelist = []
1093         if dataset:
1094             for section in dataset.sections():
1095                 if dataset.has_option(section, "station"):
1096                     stationlist.append(
1097                         eval( dataset.get(section, "station") )[1]
1098                     )
1099                 if dataset.has_option(section, "zone"):
1100                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1101         stationlist.sort()
1102         zonelist.sort()
1103         scount = len(stationlist)
1104         zcount = len(zonelist)
1105         sranks = []
1106         zranks = []
1107         for score in scores:
1108             if stationlist:
1109                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1110             if zonelist:
1111                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1112         description = search[1]
1113         uris["description"] = description
1114         print(
1115             "%s\n%s" % ( description, "-" * len(description) )
1116         )
1117         print(
1118             "%s: %s" % (
1119                 station[0],
1120                 stations.get( station[0], "description" )
1121             )
1122         )
1123         km = radian_to_km*station[1]
1124         mi = radian_to_mi*station[1]
1125         if sranks and not description.startswith("ICAO station code "):
1126             for index in range(0, len(scores)):
1127                 if station[1] >= sranks[index]:
1128                     score = scores[index][1]
1129                     break
1130             print(
1131                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1132             )
1133         elif searchtype == "coordinates":
1134             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1135         if zone[0]:
1136             print(
1137                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1138             )
1139         km = radian_to_km*zone[1]
1140         mi = radian_to_mi*zone[1]
1141         if zranks and not description.startswith("NWS/NOAA weather zone "):
1142             for index in range(0, len(scores)):
1143                 if zone[1] >= zranks[index]:
1144                     score = scores[index][1]
1145                     break
1146             print(
1147                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1148             )
1149         elif searchtype == "coordinates" and zone[0]:
1150             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1151     if cache_search:
1152         now = time.time()
1153         nowstamp = "%s (%s)" % (
1154             now,
1155             datetime.datetime.isoformat(
1156                 datetime.datetime.fromtimestamp(now),
1157                 " "
1158             )
1159         )
1160         search_cache = ["\n"]
1161         search_cache.append( "[%s]\n" % search[0] )
1162         search_cache.append( "cached = %s\n" % nowstamp )
1163         for uriname in sorted(uris.keys()):
1164             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1165         real_cachedir = os.path.expanduser(cachedir)
1166         if not os.path.exists(real_cachedir):
1167             try: os.makedirs(real_cachedir)
1168             except (IOError, OSError): pass
1169         scache_fn = os.path.join(real_cachedir, "searches")
1170         if not os.path.exists(scache_fn):
1171             then = sorted(
1172                     [ x[1] for x in datafiles.values() ],
1173                     reverse=True
1174                 )[0]
1175             thenstamp = "%s (%s)" % (
1176                 then,
1177                 datetime.datetime.isoformat(
1178                     datetime.datetime.fromtimestamp(then),
1179                     " "
1180                 )
1181             )
1182             search_cache.insert(
1183                 0,
1184                 "# based on data files from: %s\n" % thenstamp
1185             )
1186         try:
1187             scache_existing = configparser.ConfigParser()
1188             if pyversion("3"):
1189                 scache_existing.read(scache_fn, encoding="utf-8")
1190             else:
1191                 scache_existing.read(scache_fn)
1192             if not scache_existing.has_section(search[0]):
1193                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1194                 scache_fd.writelines(search_cache)
1195                 scache_fd.close()
1196         except (IOError, OSError): pass
1197     if not info:
1198         return(uris)
1199
1200 def closest(position, nodes, fieldname, angle=None):
1201     import math
1202     if not angle: angle = 2*math.pi
1203     match = None
1204     for name in nodes:
1205         if fieldname in nodes[name]:
1206             node = nodes[name][fieldname]
1207             if node and abs( position[0]-node[0] ) < angle:
1208                 if abs( position[1]-node[1] ) < angle \
1209                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1210                     if position == node:
1211                         angle = 0
1212                         match = name
1213                     else:
1214                         candidate = math.acos(
1215                             math.sin( position[0] ) * math.sin( node[0] ) \
1216                                 + math.cos( position[0] ) \
1217                                 * math.cos( node[0] ) \
1218                                 * math.cos( position[1] - node[1] )
1219                             )
1220                         if candidate < angle:
1221                             angle = candidate
1222                             match = name
1223     if match: match = str(match)
1224     return (match, angle)
1225
1226 def gecos(formatted):
1227     import math, re
1228     coordinates = formatted.split(",")
1229     for coordinate in range(0, 2):
1230         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1231             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1232             coordinates[coordinate].strip().lower()
1233         ).groups()
1234         value = float(degrees)
1235         if minutes: value += float(minutes)/60
1236         if seconds: value += float(seconds)/3600
1237         if hemisphere and hemisphere in "sw": value *= -1
1238         coordinates[coordinate] = math.radians(value)
1239     return tuple(coordinates)
1240
1241 def correlate():
1242     import codecs, csv, datetime, hashlib, os, re, sys, time, zipfile, zoneinfo
1243     if pyversion("3"): import configparser
1244     else: import ConfigParser as configparser
1245     for filename in os.listdir("."):
1246         if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename):
1247             gcounties_an = filename
1248             gcounties_fn = filename[:-4] + ".txt"
1249         elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename):
1250             gcousubs_an = filename
1251             gcousubs_fn = filename[:-4] + ".txt"
1252         elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename):
1253             gplace_an = filename
1254             gplace_fn = filename[:-4] + ".txt"
1255         elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename):
1256             gzcta_an = filename
1257             gzcta_fn = filename[:-4] + ".txt"
1258         elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename):
1259             cpfzcf_fn = filename
1260     nsdcccc_fn = "nsd_cccc.txt"
1261     ourairports_fn = "airports.csv"
1262     overrides_fn = "overrides.conf"
1263     overrideslog_fn = "overrides.log"
1264     slist_fn = "slist"
1265     zlist_fn = "zlist"
1266     qalog_fn = "qa.log"
1267     airports_fn = "airports"
1268     places_fn = "places"
1269     stations_fn = "stations"
1270     zctas_fn = "zctas"
1271     zones_fn = "zones"
1272     header = """\
1273 %s
1274 # generated by %s on %s from these public domain sources:
1275 #
1276 # https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html
1277 # %s %s %s
1278 # %s %s %s
1279 # %s %s %s
1280 # %s %s %s
1281 #
1282 # https://www.weather.gov/gis/ZoneCounty/
1283 # %s %s %s
1284 #
1285 # https://tgftp.nws.noaa.gov/data/
1286 # %s %s %s
1287 #
1288 # https://ourairports.com/data/
1289 # %s %s %s
1290 #
1291 # ...and these manually-generated or hand-compiled adjustments:
1292 # %s %s %s
1293 # %s %s %s
1294 # %s %s %s\
1295 """ % (
1296         weather_copyright,
1297         os.path.basename( sys.argv[0] ),
1298         datetime.date.isoformat(
1299             datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
1300         ),
1301         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1302         datetime.date.isoformat(
1303             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
1304         ),
1305         gcounties_an,
1306         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1307         datetime.date.isoformat(
1308             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
1309         ),
1310         gcousubs_an,
1311         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
1312         datetime.date.isoformat(
1313             datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
1314         ),
1315         gplace_an,
1316         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1317         datetime.date.isoformat(
1318             datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
1319         ),
1320         gzcta_an,
1321         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1322         datetime.date.isoformat(
1323             datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
1324         ),
1325         cpfzcf_fn,
1326         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1327         datetime.date.isoformat(
1328             datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
1329         ),
1330         nsdcccc_fn,
1331         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
1332         datetime.date.isoformat(
1333             datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
1334         ),
1335         ourairports_fn,
1336         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1337         datetime.date.isoformat(
1338             datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
1339         ),
1340         overrides_fn,
1341         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1342         datetime.date.isoformat(
1343             datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
1344         ),
1345         slist_fn,
1346         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1347         datetime.date.isoformat(
1348             datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
1349         ),
1350         zlist_fn
1351     )
1352     airports = {}
1353     places = {}
1354     stations = {}
1355     zctas = {}
1356     zones = {}
1357     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1358     sys.stdout.write(message)
1359     sys.stdout.flush()
1360     count = 0
1361     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
1362     columns = gcounties.readline().decode("utf-8").strip().split("\t")
1363     for line in gcounties:
1364         fields = line.decode("utf-8").strip().split("\t")
1365         f_geoid = fields[ columns.index("GEOID") ].strip()
1366         f_name = fields[ columns.index("NAME") ].strip()
1367         f_usps = fields[ columns.index("USPS") ].strip()
1368         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1369         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1370         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1371             fips = "fips%s" % f_geoid
1372             if fips not in places: places[fips] = {}
1373             places[fips]["centroid"] = gecos(
1374                 "%s,%s" % (f_intptlat, f_intptlong)
1375             )
1376             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1377             count += 1
1378     gcounties.close()
1379     print("done (%s lines)." % count)
1380     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1381     sys.stdout.write(message)
1382     sys.stdout.flush()
1383     count = 0
1384     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
1385     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
1386     for line in gcousubs:
1387         fields = line.decode("utf-8").strip().split("\t")
1388         f_geoid = fields[ columns.index("GEOID") ].strip()
1389         f_name = fields[ columns.index("NAME") ].strip()
1390         f_usps = fields[ columns.index("USPS") ].strip()
1391         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1392         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1393         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1394             fips = "fips%s" % f_geoid
1395             if fips not in places: places[fips] = {}
1396             places[fips]["centroid"] = gecos(
1397                 "%s,%s" % (f_intptlat, f_intptlong)
1398             )
1399             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1400             count += 1
1401     gcousubs.close()
1402     print("done (%s lines)." % count)
1403     message = "Reading %s:%s..." % (gplace_an, gplace_fn)
1404     sys.stdout.write(message)
1405     sys.stdout.flush()
1406     count = 0
1407     gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
1408     columns = gplace.readline().decode("utf-8").strip().split("\t")
1409     for line in gplace:
1410         fields = line.decode("utf-8").strip().split("\t")
1411         f_geoid = fields[ columns.index("GEOID") ].strip()
1412         f_name = fields[ columns.index("NAME") ].strip()
1413         f_usps = fields[ columns.index("USPS") ].strip()
1414         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1415         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1416         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1417             fips = "fips%s" % f_geoid
1418             if fips not in places: places[fips] = {}
1419             places[fips]["centroid"] = gecos(
1420                 "%s,%s" % (f_intptlat, f_intptlong)
1421             )
1422             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1423             count += 1
1424     gplace.close()
1425     print("done (%s lines)." % count)
1426     message = "Reading %s..." % slist_fn
1427     sys.stdout.write(message)
1428     sys.stdout.flush()
1429     count = 0
1430     slist = codecs.open(slist_fn, "r", "utf-8")
1431     for line in slist:
1432         icao = line.split("#")[0].strip()
1433         if icao:
1434             stations[icao] = {
1435                 "metar": "https://tgftp.nws.noaa.gov/data/observations/"\
1436                     + "metar/decoded/%s.TXT" % icao.upper()
1437             }
1438             count += 1
1439     slist.close()
1440     print("done (%s lines)." % count)
1441     message = "Reading %s..." % nsdcccc_fn
1442     sys.stdout.write(message)
1443     sys.stdout.flush()
1444     count = 0
1445     nsdcccc = codecs.open(nsdcccc_fn, "r", "utf-8")
1446     for line in nsdcccc:
1447         line = str(line)
1448         fields = line.split(";")
1449         icao = fields[0].strip().lower()
1450         if icao in stations:
1451             description = []
1452             name = " ".join( fields[3].strip().title().split() )
1453             if name: description.append(name)
1454             st = fields[4].strip()
1455             if st: description.append(st)
1456             country = " ".join( fields[5].strip().title().split() )
1457             if country: description.append(country)
1458             if description:
1459                 stations[icao]["description"] = ", ".join(description)
1460             lat, lon = fields[7:9]
1461             if lat and lon:
1462                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1463             elif "location" not in stations[icao]:
1464                 lat, lon = fields[5:7]
1465                 if lat and lon:
1466                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1467         count += 1
1468     nsdcccc.close()
1469     print("done (%s lines)." % count)
1470     message = "Reading %s..." % ourairports_fn
1471     sys.stdout.write(message)
1472     sys.stdout.flush()
1473     count = 0
1474     ourairports = open(ourairports_fn, "r")
1475     for row in csv.reader(ourairports):
1476         icao = row[12].lower()
1477         if icao in stations:
1478             iata = row[13].lower()
1479             if len(iata) == 3: airports[iata] = { "station": icao }
1480             if "description" not in stations[icao]:
1481                 description = []
1482                 name = row[3]
1483                 if name: description.append(name)
1484                 municipality = row[10]
1485                 if municipality: description.append(municipality)
1486                 region = row[9]
1487                 country = row[8]
1488                 if region:
1489                     if "-" in region:
1490                         c,r = region.split("-", 1)
1491                         if c == country: region = r
1492                     description.append(region)
1493                 if country:
1494                     description.append(country)
1495                 if description:
1496                     stations[icao]["description"] = ", ".join(description)
1497             if "location" not in stations[icao]:
1498                 lat = row[4]
1499                 if lat:
1500                     lon = row[5]
1501                     if lon:
1502                         stations[icao]["location"] = gecos(
1503                             "%s,%s" % (lat, lon)
1504                         )
1505         count += 1
1506     ourairports.close()
1507     print("done (%s lines)." % count)
1508     message = "Reading %s..." % zlist_fn
1509     sys.stdout.write(message)
1510     sys.stdout.flush()
1511     count = 0
1512     zlist = codecs.open(zlist_fn, "r", "utf-8")
1513     for line in zlist:
1514         line = line.split("#")[0].strip()
1515         if line:
1516             zones[line] = {}
1517             count += 1
1518     zlist.close()
1519     print("done (%s lines)." % count)
1520     message = "Reading %s..." % cpfzcf_fn
1521     sys.stdout.write(message)
1522     sys.stdout.flush()
1523     count = 0
1524     cpfz = {}
1525     cpfzcf = codecs.open(cpfzcf_fn, "r", "utf-8")
1526     for line in cpfzcf:
1527         fields = line.strip().split("|")
1528         if len(fields) == 11 \
1529             and fields[0] and fields[1] and fields[9] and fields[10]:
1530             zone = "z".join( fields[:2] ).lower()
1531             if zone in zones:
1532                 state = fields[0]
1533                 description = fields[3].strip()
1534                 fips = "fips%s"%fields[6]
1535                 countycode = "%sc%s" % (state.lower(), fips[-3:])
1536                 if state:
1537                     zones[zone]["coastal_flood_statement"] = (
1538                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1539                         "flood/coastal/%s/%s.txt" % (state.lower(), zone))
1540                     zones[zone]["flash_flood_statement"] = (
1541                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1542                         "flash_flood/statement/%s/%s.txt"
1543                         % (state.lower(), countycode))
1544                     zones[zone]["flash_flood_warning"] = (
1545                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1546                         "flash_flood/warning/%s/%s.txt"
1547                         % (state.lower(), countycode))
1548                     zones[zone]["flash_flood_watch"] = (
1549                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1550                         "flash_flood/watch/%s/%s.txt" % (state.lower(), zone))
1551                     zones[zone]["flood_warning"] = (
1552                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1553                         "flood/warning/%s/%s.txt"
1554                         % (state.lower(), countycode))
1555                     zones[zone]["severe_thunderstorm_warning"] = (
1556                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1557                         "thunderstorm/%s/%s.txt" % (state.lower(), countycode))
1558                     zones[zone]["severe_weather_statement"] = (
1559                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1560                         "severe_weather_stmt/%s/%s.txt"
1561                         % (state.lower(), countycode))
1562                     zones[zone]["short_term_forecast"] = (
1563                         "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/"
1564                         "%s/%s.txt" % (state.lower(), zone))
1565                     zones[zone]["special_weather_statement"] = (
1566                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1567                         "special_weather_stmt/%s/%s.txt"
1568                         % (state.lower(), zone))
1569                     zones[zone]["state_forecast"] = (
1570                         "https://tgftp.nws.noaa.gov/data/forecasts/state/"
1571                         "%s/%s.txt" % (state.lower(), zone))
1572                     zones[zone]["tornado"] = (
1573                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1574                         "tornado/%s/%s.txt" % (state.lower(), countycode))
1575                     zones[zone]["urgent_weather_message"] = (
1576                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1577                         "non_precip/%s/%s.txt" % (state.lower(), zone))
1578                     zones[zone]["zone_forecast"] = (
1579                         "https://tgftp.nws.noaa.gov/data/forecasts/zone/"
1580                         "%s/%s.txt" % (state.lower(), zone))
1581                 tzcode = fields[7]
1582                 if tzcode == "A":
1583                     zones[zone]["tz"] = "US/Alaska"
1584                 elif tzcode == "AH":
1585                     zones[zone]["tz"] = "US/Aleutian"
1586                 elif tzcode in ("C", "CE", "CM"):
1587                     zones[zone]["tz"] = "US/Central"
1588                 elif tzcode in ("E", "e"):
1589                     zones[zone]["tz"] = "US/Eastern"
1590                 elif tzcode == "F":
1591                     zones[zone]["tz"] = "Pacific/Guadalcanal"
1592                 elif tzcode == "G":
1593                     zones[zone]["tz"] = "Pacific/Guam"
1594                 elif tzcode == "H":
1595                     zones[zone]["tz"] = "US/Hawaii"
1596                 elif tzcode == "J":
1597                     zones[zone]["tz"] = "Japan"
1598                 elif tzcode == "K":
1599                     zones[zone]["tz"] = "Pacific/Kwajalein"
1600                 elif tzcode in ("M", "MC", "MP"):
1601                     zones[zone]["tz"] = "US/Mountain"
1602                 elif tzcode == "m":
1603                     zones[zone]["tz"] = "US/Arizona"
1604                 elif tzcode == "P":
1605                     zones[zone]["tz"] = "US/Pacific"
1606                 elif tzcode == "S":
1607                     zones[zone]["tz"] = "US/Samoa"
1608                 elif tzcode == "V":
1609                     zones[zone]["tz"] = "America/Virgin"
1610                 else:
1611                     zones[zone]["tz"] = ""
1612                 county = fields[5]
1613                 if county:
1614                     if description.endswith(county):
1615                         description += " County"
1616                     else:
1617                         description += ", %s County" % county
1618                 description += ", %s, US" % state
1619                 zones[zone]["description"] = description
1620                 zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
1621                 if fips in places and not zones[zone]["centroid"]:
1622                     zones[zone]["centroid"] = places[fips]["centroid"]
1623         count += 1
1624     cpfzcf.close()
1625     print("done (%s lines)." % count)
1626     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1627     sys.stdout.write(message)
1628     sys.stdout.flush()
1629     count = 0
1630     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
1631     columns = gzcta.readline().decode("utf-8").strip().split("\t")
1632     for line in gzcta:
1633         fields = line.decode("utf-8").strip().split("\t")
1634         f_geoid = fields[ columns.index("GEOID") ].strip()
1635         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1636         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1637         if f_geoid and f_intptlat and f_intptlong:
1638             if f_geoid not in zctas: zctas[f_geoid] = {}
1639             zctas[f_geoid]["centroid"] = gecos(
1640                 "%s,%s" % (f_intptlat, f_intptlong)
1641             )
1642             count += 1
1643     gzcta.close()
1644     print("done (%s lines)." % count)
1645     message = "Reading %s..." % overrides_fn
1646     sys.stdout.write(message)
1647     sys.stdout.flush()
1648     count = 0
1649     added = 0
1650     removed = 0
1651     changed = 0
1652     overrides = configparser.ConfigParser()
1653     overrides.read_file( codecs.open(overrides_fn, "r", "utf8") )
1654     overrideslog = []
1655     for section in overrides.sections():
1656         addopt = 0
1657         chgopt = 0
1658         if section.startswith("-"):
1659             section = section[1:]
1660             delete = True
1661         else: delete = False
1662         if re.match("[A-Za-z]{3}$", section):
1663             if delete:
1664                 if section in airports:
1665                     del( airports[section] )
1666                     logact = "removed airport %s" % section
1667                     removed += 1
1668                 else:
1669                     logact = "tried to remove nonexistent airport %s" % section
1670             else:
1671                 if section in airports:
1672                     logact = "changed airport %s" % section
1673                     changed += 1
1674                 else:
1675                     airports[section] = {}
1676                     logact = "added airport %s" % section
1677                     added += 1
1678                 for key,value in overrides.items(section):
1679                     if key in airports[section]: chgopt += 1
1680                     else: addopt += 1
1681                     if key in ("centroid", "location"):
1682                         airports[section][key] = eval(value)
1683                     else:
1684                         airports[section][key] = value
1685                 if addopt and chgopt:
1686                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1687                 elif addopt: logact += " (+%s options)" % addopt
1688                 elif chgopt: logact += " (!%s options)" % chgopt
1689         elif re.match("[A-Za-z0-9]{4}$", section):
1690             if delete:
1691                 if section in stations:
1692                     del( stations[section] )
1693                     logact = "removed station %s" % section
1694                     removed += 1
1695                 else:
1696                     logact = "tried to remove nonexistent station %s" % section
1697             else:
1698                 if section in stations:
1699                     logact = "changed station %s" % section
1700                     changed += 1
1701                 else:
1702                     stations[section] = {}
1703                     logact = "added station %s" % section
1704                     added += 1
1705                 for key,value in overrides.items(section):
1706                     if key in stations[section]: chgopt += 1
1707                     else: addopt += 1
1708                     if key in ("centroid", "location"):
1709                         stations[section][key] = eval(value)
1710                     else:
1711                         stations[section][key] = value
1712                 if addopt and chgopt:
1713                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1714                 elif addopt: logact += " (+%s options)" % addopt
1715                 elif chgopt: logact += " (!%s options)" % chgopt
1716         elif re.match("[0-9]{5}$", section):
1717             if delete:
1718                 if section in zctas:
1719                     del( zctas[section] )
1720                     logact = "removed zcta %s" % section
1721                     removed += 1
1722                 else:
1723                     logact = "tried to remove nonexistent zcta %s" % section
1724             else:
1725                 if section in zctas:
1726                     logact = "changed zcta %s" % section
1727                     changed += 1
1728                 else:
1729                     zctas[section] = {}
1730                     logact = "added zcta %s" % section
1731                     added += 1
1732                 for key,value in overrides.items(section):
1733                     if key in zctas[section]: chgopt += 1
1734                     else: addopt += 1
1735                     if key in ("centroid", "location"):
1736                         zctas[section][key] = eval(value)
1737                     else:
1738                         zctas[section][key] = value
1739                 if addopt and chgopt:
1740                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1741                 elif addopt: logact += " (+%s options)" % addopt
1742                 elif chgopt: logact += " (!%s options)" % chgopt
1743         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1744             if delete:
1745                 if section in zones:
1746                     del( zones[section] )
1747                     logact = "removed zone %s" % section
1748                     removed += 1
1749                 else:
1750                     logact = "tried to remove nonexistent zone %s" % section
1751             else:
1752                 if section in zones:
1753                     logact = "changed zone %s" % section
1754                     changed += 1
1755                 else:
1756                     zones[section] = {}
1757                     logact = "added zone %s" % section
1758                     added += 1
1759                 for key,value in overrides.items(section):
1760                     if key in zones[section]: chgopt += 1
1761                     else: addopt += 1
1762                     if key in ("centroid", "location"):
1763                         zones[section][key] = eval(value)
1764                     else:
1765                         zones[section][key] = value
1766                 if addopt and chgopt:
1767                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1768                 elif addopt: logact += " (+%s options)" % addopt
1769                 elif chgopt: logact += " (!%s options)" % chgopt
1770         elif re.match("fips[0-9]+$", section):
1771             if delete:
1772                 if section in places:
1773                     del( places[section] )
1774                     logact = "removed place %s" % section
1775                     removed += 1
1776                 else:
1777                     logact = "tried to remove nonexistent place %s" % section
1778             else:
1779                 if section in places:
1780                     logact = "changed place %s" % section
1781                     changed += 1
1782                 else:
1783                     places[section] = {}
1784                     logact = "added place %s" % section
1785                     added += 1
1786                 for key,value in overrides.items(section):
1787                     if key in places[section]: chgopt += 1
1788                     else: addopt += 1
1789                     if key in ("centroid", "location"):
1790                         places[section][key] = eval(value)
1791                     else:
1792                         places[section][key] = value
1793                 if addopt and chgopt:
1794                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1795                 elif addopt: logact += " (+%s options)" % addopt
1796                 elif chgopt: logact += " (!%s options)" % chgopt
1797         count += 1
1798         overrideslog.append("%s\n" % logact)
1799     overrideslog.sort()
1800     if os.path.exists(overrideslog_fn):
1801         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1802     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1803     import time
1804     overrideslog_fd.write(
1805         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
1806         '# use, copy, modify, and distribute this software is granted under terms\n'
1807         '# provided in the LICENSE file distributed with this software.\n\n'
1808         % time.gmtime().tm_year)
1809     overrideslog_fd.writelines(overrideslog)
1810     overrideslog_fd.close()
1811     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1812         count,
1813         added,
1814         removed,
1815         changed
1816     ) )
1817     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1818     print(
1819         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1820             estimate
1821     )
1822     count = 0
1823     milestones = list( range(51) )
1824     message = "   "
1825     sys.stdout.write(message)
1826     sys.stdout.flush()
1827     for fips in places:
1828         centroid = places[fips]["centroid"]
1829         if centroid:
1830             station = closest(centroid, stations, "location", 0.1)
1831         if station[0]:
1832             places[fips]["station"] = station
1833             count += 1
1834             if not count%100:
1835                 level = int(50*count/estimate)
1836                 if level in milestones:
1837                     for remaining in milestones[:milestones.index(level)+1]:
1838                         if remaining%5:
1839                             message = "."
1840                             sys.stdout.write(message)
1841                             sys.stdout.flush()
1842                         else:
1843                             message = "%s%%" % (remaining*2,)
1844                             sys.stdout.write(message)
1845                             sys.stdout.flush()
1846                         milestones.remove(remaining)
1847         if centroid:
1848             zone = closest(centroid, zones, "centroid", 0.1)
1849         if zone[0]:
1850             places[fips]["zone"] = zone
1851             count += 1
1852             if not count%100:
1853                 level = int(50*count/estimate)
1854                 if level in milestones:
1855                     for remaining in milestones[:milestones.index(level)+1]:
1856                         if remaining%5:
1857                             message = "."
1858                             sys.stdout.write(message)
1859                             sys.stdout.flush()
1860                         else:
1861                             message = "%s%%" % (remaining*2,)
1862                             sys.stdout.write(message)
1863                             sys.stdout.flush()
1864                         milestones.remove(remaining)
1865     for station in stations:
1866         if "location" in stations[station]:
1867             location = stations[station]["location"]
1868             if location:
1869                 zone = closest(location, zones, "centroid", 0.1)
1870             if zone[0]:
1871                 stations[station]["zone"] = zone
1872                 count += 1
1873                 if not count%100:
1874                     level = int(50*count/estimate)
1875                     if level in milestones:
1876                         for remaining in milestones[:milestones.index(level)+1]:
1877                             if remaining%5:
1878                                 message = "."
1879                                 sys.stdout.write(message)
1880                                 sys.stdout.flush()
1881                             else:
1882                                 message = "%s%%" % (remaining*2,)
1883                                 sys.stdout.write(message)
1884                                 sys.stdout.flush()
1885                             milestones.remove(remaining)
1886     for zcta in zctas.keys():
1887         centroid = zctas[zcta]["centroid"]
1888         if centroid:
1889             station = closest(centroid, stations, "location", 0.1)
1890         if station[0]:
1891             zctas[zcta]["station"] = station
1892             count += 1
1893             if not count%100:
1894                 level = int(50*count/estimate)
1895                 if level in milestones:
1896                     for remaining in milestones[ : milestones.index(level)+1 ]:
1897                         if remaining%5:
1898                             message = "."
1899                             sys.stdout.write(message)
1900                             sys.stdout.flush()
1901                         else:
1902                             message = "%s%%" % (remaining*2,)
1903                             sys.stdout.write(message)
1904                             sys.stdout.flush()
1905                         milestones.remove(remaining)
1906         if centroid:
1907             zone = closest(centroid, zones, "centroid", 0.1)
1908         if zone[0]:
1909             zctas[zcta]["zone"] = zone
1910             count += 1
1911             if not count%100:
1912                 level = int(50*count/estimate)
1913                 if level in milestones:
1914                     for remaining in milestones[:milestones.index(level)+1]:
1915                         if remaining%5:
1916                             message = "."
1917                             sys.stdout.write(message)
1918                             sys.stdout.flush()
1919                         else:
1920                             message = "%s%%" % (remaining*2,)
1921                             sys.stdout.write(message)
1922                             sys.stdout.flush()
1923                         milestones.remove(remaining)
1924     for zone in zones.keys():
1925         if "centroid" in zones[zone]:
1926             centroid = zones[zone]["centroid"]
1927             if centroid:
1928                 station = closest(centroid, stations, "location", 0.1)
1929             if station[0]:
1930                 zones[zone]["station"] = station
1931                 count += 1
1932                 if not count%100:
1933                     level = int(50*count/estimate)
1934                     if level in milestones:
1935                         for remaining in milestones[:milestones.index(level)+1]:
1936                             if remaining%5:
1937                                 message = "."
1938                                 sys.stdout.write(message)
1939                                 sys.stdout.flush()
1940                             else:
1941                                 message = "%s%%" % (remaining*2,)
1942                                 sys.stdout.write(message)
1943                                 sys.stdout.flush()
1944                             milestones.remove(remaining)
1945     for remaining in milestones:
1946         if remaining%5:
1947             message = "."
1948             sys.stdout.write(message)
1949             sys.stdout.flush()
1950         else:
1951             message = "%s%%" % (remaining*2,)
1952             sys.stdout.write(message)
1953             sys.stdout.flush()
1954     print("\n   done (%s correlations)." % count)
1955     message = "Writing %s..." % airports_fn
1956     sys.stdout.write(message)
1957     sys.stdout.flush()
1958     count = 0
1959     if os.path.exists(airports_fn):
1960         os.rename(airports_fn, "%s_old"%airports_fn)
1961     airports_fd = codecs.open(airports_fn, "w", "utf8")
1962     airports_fd.write(header)
1963     for airport in sorted( airports.keys() ):
1964         airports_fd.write("\n\n[%s]" % airport)
1965         for key, value in sorted( airports[airport].items() ):
1966             if type(value) is float: value = "%.7f"%value
1967             elif type(value) is tuple:
1968                 elements = []
1969                 for element in value:
1970                     if type(element) is float: elements.append("%.7f"%element)
1971                     else: elements.append( repr(element) )
1972                 value = "(%s)"%", ".join(elements)
1973             airports_fd.write( "\n%s = %s" % (key, value) )
1974         count += 1
1975     airports_fd.write("\n")
1976     airports_fd.close()
1977     print("done (%s sections)." % count)
1978     message = "Writing %s..." % places_fn
1979     sys.stdout.write(message)
1980     sys.stdout.flush()
1981     count = 0
1982     if os.path.exists(places_fn):
1983         os.rename(places_fn, "%s_old"%places_fn)
1984     places_fd = codecs.open(places_fn, "w", "utf8")
1985     places_fd.write(header)
1986     for fips in sorted( places.keys() ):
1987         places_fd.write("\n\n[%s]" % fips)
1988         for key, value in sorted( places[fips].items() ):
1989             if type(value) is float: value = "%.7f"%value
1990             elif type(value) is tuple:
1991                 elements = []
1992                 for element in value:
1993                     if type(element) is float: elements.append("%.7f"%element)
1994                     else: elements.append( repr(element) )
1995                 value = "(%s)"%", ".join(elements)
1996             places_fd.write( "\n%s = %s" % (key, value) )
1997         count += 1
1998     places_fd.write("\n")
1999     places_fd.close()
2000     print("done (%s sections)." % count)
2001     message = "Writing %s..." % stations_fn
2002     sys.stdout.write(message)
2003     sys.stdout.flush()
2004     count = 0
2005     if os.path.exists(stations_fn):
2006         os.rename(stations_fn, "%s_old"%stations_fn)
2007     stations_fd = codecs.open(stations_fn, "w", "utf-8")
2008     stations_fd.write(header)
2009     for station in sorted( stations.keys() ):
2010         stations_fd.write("\n\n[%s]" % station)
2011         for key, value in sorted( stations[station].items() ):
2012             if type(value) is float: value = "%.7f"%value
2013             elif type(value) is tuple:
2014                 elements = []
2015                 for element in value:
2016                     if type(element) is float: elements.append("%.7f"%element)
2017                     else: elements.append( repr(element) )
2018                 value = "(%s)"%", ".join(elements)
2019             if type(value) is bytes:
2020                 value = value.decode("utf-8")
2021             stations_fd.write( "\n%s = %s" % (key, value) )
2022         count += 1
2023     stations_fd.write("\n")
2024     stations_fd.close()
2025     print("done (%s sections)." % count)
2026     message = "Writing %s..." % zctas_fn
2027     sys.stdout.write(message)
2028     sys.stdout.flush()
2029     count = 0
2030     if os.path.exists(zctas_fn):
2031         os.rename(zctas_fn, "%s_old"%zctas_fn)
2032     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
2033     zctas_fd.write(header)
2034     for zcta in sorted( zctas.keys() ):
2035         zctas_fd.write("\n\n[%s]" % zcta)
2036         for key, value in sorted( zctas[zcta].items() ):
2037             if type(value) is float: value = "%.7f"%value
2038             elif type(value) is tuple:
2039                 elements = []
2040                 for element in value:
2041                     if type(element) is float: elements.append("%.7f"%element)
2042                     else: elements.append( repr(element) )
2043                 value = "(%s)"%", ".join(elements)
2044             zctas_fd.write( "\n%s = %s" % (key, value) )
2045         count += 1
2046     zctas_fd.write("\n")
2047     zctas_fd.close()
2048     print("done (%s sections)." % count)
2049     message = "Writing %s..." % zones_fn
2050     sys.stdout.write(message)
2051     sys.stdout.flush()
2052     count = 0
2053     if os.path.exists(zones_fn):
2054         os.rename(zones_fn, "%s_old"%zones_fn)
2055     zones_fd = codecs.open(zones_fn, "w", "utf8")
2056     zones_fd.write(header)
2057     for zone in sorted( zones.keys() ):
2058         zones_fd.write("\n\n[%s]" % zone)
2059         for key, value in sorted( zones[zone].items() ):
2060             if type(value) is float: value = "%.7f"%value
2061             elif type(value) is tuple:
2062                 elements = []
2063                 for element in value:
2064                     if type(element) is float: elements.append("%.7f"%element)
2065                     else: elements.append( repr(element) )
2066                 value = "(%s)"%", ".join(elements)
2067             zones_fd.write( "\n%s = %s" % (key, value) )
2068         count += 1
2069     zones_fd.write("\n")
2070     zones_fd.close()
2071     print("done (%s sections)." % count)
2072     message = "Starting QA check..."
2073     sys.stdout.write(message)
2074     sys.stdout.flush()
2075     airports = configparser.ConfigParser()
2076     if pyversion("3"):
2077         airports.read(airports_fn, encoding="utf-8")
2078     else:
2079         airports.read(airports_fn)
2080     places = configparser.ConfigParser()
2081     if pyversion("3"):
2082         places.read(places_fn, encoding="utf-8")
2083     else:
2084         places.read(places_fn)
2085     stations = configparser.ConfigParser()
2086     if pyversion("3"):
2087         stations.read(stations_fn, encoding="utf-8")
2088     else:
2089         stations.read(stations_fn)
2090     zctas = configparser.ConfigParser()
2091     if pyversion("3"):
2092         zctas.read(zctas_fn, encoding="utf-8")
2093     else:
2094         zctas.read(zctas_fn)
2095     zones = configparser.ConfigParser()
2096     if pyversion("3"):
2097         zones.read(zones_fn, encoding="utf-8")
2098     else:
2099         zones.read(zones_fn)
2100     qalog = []
2101     places_nocentroid = 0
2102     places_nodescription = 0
2103     for place in sorted( places.sections() ):
2104         if not places.has_option(place, "centroid"):
2105             qalog.append("%s: no centroid\n" % place)
2106             places_nocentroid += 1
2107         if not places.has_option(place, "description"):
2108             qalog.append("%s: no description\n" % place)
2109             places_nodescription += 1
2110     stations_nodescription = 0
2111     stations_nolocation = 0
2112     stations_nometar = 0
2113     for station in sorted( stations.sections() ):
2114         if not stations.has_option(station, "description"):
2115             qalog.append("%s: no description\n" % station)
2116             stations_nodescription += 1
2117         if not stations.has_option(station, "location"):
2118             qalog.append("%s: no location\n" % station)
2119             stations_nolocation += 1
2120         if not stations.has_option(station, "metar"):
2121             qalog.append("%s: no metar\n" % station)
2122             stations_nometar += 1
2123     airports_badstation = 0
2124     airports_nostation = 0
2125     for airport in sorted( airports.sections() ):
2126         if not airports.has_option(airport, "station"):
2127             qalog.append("%s: no station\n" % airport)
2128             airports_nostation += 1
2129         else:
2130             station = airports.get(airport, "station")
2131             if station not in stations.sections():
2132                 qalog.append( "%s: bad station %s\n" % (airport, station) )
2133                 airports_badstation += 1
2134     zctas_nocentroid = 0
2135     for zcta in sorted( zctas.sections() ):
2136         if not zctas.has_option(zcta, "centroid"):
2137             qalog.append("%s: no centroid\n" % zcta)
2138             zctas_nocentroid += 1
2139     zones_nocentroid = 0
2140     zones_nodescription = 0
2141     zones_notz = 0
2142     zones_noforecast = 0
2143     zones_overlapping = 0
2144     zonetable = {}
2145     for zone in zones.sections():
2146         if zones.has_option(zone, "centroid"):
2147             zonetable[zone] = {
2148                 "centroid": eval( zones.get(zone, "centroid") )
2149             }
2150     for zone in sorted( zones.sections() ):
2151         if zones.has_option(zone, "centroid"):
2152             zonetable_local = zonetable.copy()
2153             del( zonetable_local[zone] )
2154             centroid = eval( zones.get(zone, "centroid") )
2155             if centroid:
2156                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2157             if nearest[1]*radian_to_km < 1:
2158                 qalog.append( "%s: within one km of %s\n" % (
2159                     zone,
2160                     nearest[0]
2161                 ) )
2162                 zones_overlapping += 1
2163         else:
2164             qalog.append("%s: no centroid\n" % zone)
2165             zones_nocentroid += 1
2166         if not zones.has_option(zone, "description"):
2167             qalog.append("%s: no description\n" % zone)
2168             zones_nodescription += 1
2169         if not zones.has_option(zone, "tz") or not zones.get(
2170                 zone, "tz") in zoneinfo.available_timezones():
2171             qalog.append("%s: no time zone\n" % zone)
2172             zones_notz += 1
2173         if not zones.has_option(zone, "zone_forecast"):
2174             qalog.append("%s: no forecast\n" % zone)
2175             zones_noforecast += 1
2176     if os.path.exists(qalog_fn):
2177         os.rename(qalog_fn, "%s_old"%qalog_fn)
2178     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2179     import time
2180     qalog_fd.write(
2181         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
2182         '# use, copy, modify, and distribute this software is granted under terms\n'
2183         '# provided in the LICENSE file distributed with this software.\n\n'
2184         % time.gmtime().tm_year)
2185     qalog_fd.writelines(qalog)
2186     qalog_fd.close()
2187     if qalog:
2188         print("issues found (see %s for details):"%qalog_fn)
2189         if airports_badstation:
2190             print("   %s airports with invalid station"%airports_badstation)
2191         if airports_nostation:
2192             print("   %s airports with no station"%airports_nostation)
2193         if places_nocentroid:
2194             print("   %s places with no centroid"%places_nocentroid)
2195         if places_nodescription:
2196             print("   %s places with no description"%places_nodescription)
2197         if stations_nodescription:
2198             print("   %s stations with no description"%stations_nodescription)
2199         if stations_nolocation:
2200             print("   %s stations with no location"%stations_nolocation)
2201         if stations_nometar:
2202             print("   %s stations with no METAR"%stations_nometar)
2203         if zctas_nocentroid:
2204             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2205         if zones_nocentroid:
2206             print("   %s zones with no centroid"%zones_nocentroid)
2207         if zones_nodescription:
2208             print("   %s zones with no description"%zones_nodescription)
2209         if zones_notz:
2210             print("   %s zones with no time zone"%zones_notz)
2211         if zones_noforecast:
2212             print("   %s zones with no forecast"%zones_noforecast)
2213         if zones_overlapping:
2214             print("   %s zones within one km of another"%zones_overlapping)
2215     else: print("no issues found.")
2216     print("Indexing complete!")