weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2023 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.4.4"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 def pyversion(ref=None):
  15     """Determine the Python version and optionally compare to a reference."""
  16     import platform
  17     ver = platform.python_version()
  18     if ref:
  19         return [
  20             int(x) for x in ver.split(".")[:2]
  21         ] >= [
  22             int(x) for x in ref.split(".")[:2]
  23         ]
  24     else: return ver
  25
  26 class Selections:
  27     """An object to contain selection data."""
  28     def __init__(self):
  29         """Store the config, options and arguments."""
  30         self.config = get_config()
  31         self.options, self.arguments = get_options(self.config)
  32         if self.get_bool("cache") and self.get_bool("cache_search") \
  33             and not self.get_bool("longlist"):
  34             integrate_search_cache(
  35                 self.config,
  36                 self.get("cachedir"),
  37                 self.get("setpath")
  38             )
  39         if not self.arguments:
  40             if "id" in self.options.__dict__ \
  41                 and self.options.__dict__["id"]:
  42                 self.arguments.append( self.options.__dict__["id"] )
  43                 del( self.options.__dict__["id"] )
  44                 import sys
  45                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  46                 sys.stderr.write(message)
  47             elif "city" in self.options.__dict__ \
  48                 and self.options.__dict__["city"] \
  49                 and "st" in self.options.__dict__ \
  50                 and self.options.__dict__["st"]:
  51                 self.arguments.append(
  52                     "^%s city, %s" % (
  53                         self.options.__dict__["city"],
  54                         self.options.__dict__["st"]
  55                     )
  56                 )
  57                 del( self.options.__dict__["city"] )
  58                 del( self.options.__dict__["st"] )
  59                 import sys
  60                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  61                 sys.stderr.write(message)
  62     def get(self, option, argument=None):
  63         """Retrieve data from the config or options."""
  64         if argument:
  65             if self.config.has_section(argument) and (
  66                 self.config.has_option(argument, "city") \
  67                     or self.config.has_option(argument, "id") \
  68                     or self.config.has_option(argument, "st")
  69             ):
  70                 self.config.remove_section(argument)
  71                 import sys
  72                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  73                 sys.stderr.write(message)
  74             if not self.config.has_section(argument):
  75                 guessed = guess(
  76                     argument,
  77                     path=self.get("setpath"),
  78                     info=self.get("info"),
  79                     cache_search=(
  80                         self.get("cache") and self.get("cache_search")
  81                     ),
  82                     cachedir=self.get("cachedir"),
  83                     quiet=self.get_bool("quiet")
  84                 )
  85                 self.config.add_section(argument)
  86                 for item in guessed.items():
  87                     self.config.set(argument, *item)
  88             if self.config.has_option(argument, option):
  89                 return self.config.get(argument, option)
  90         if option in self.options.__dict__:
  91             return self.options.__dict__[option]
  92         import sys
  93         message = "WARNING: no URI defined for %s\n" % option
  94         sys.stderr.write(message)
  95         return None
  96     def get_bool(self, option, argument=None):
  97         """Get data and coerce to a boolean if necessary."""
  98         # Mimic configparser's getboolean() method by treating
  99         # false/no/off/0 as False and true/yes/on/1 as True values,
 100         # case-insensitively
 101         value = self.get(option, argument)
 102         if isinstance(value, bool):
 103             return value
 104         if isinstance(value, str):
 105             vlower = value.lower()
 106             if vlower in ('false', 'no', 'off', '0'):
 107                 return False
 108             elif vlower in ('true', 'yes', 'on', '1'):
 109                 return True
 110         raise ValueError("Not a boolean: %s" % value)
 111     def getint(self, option, argument=None):
 112         """Get data and coerce to an integer if necessary."""
 113         value = self.get(option, argument)
 114         if value: return int(value)
 115         else: return 0
 116
 117 def average(coords):
 118     """Average a list of coordinates."""
 119     x = 0
 120     y = 0
 121     for coord in coords:
 122         x += coord[0]
 123         y += coord[1]
 124     count = len(coords)
 125     return (x/count, y/count)
 126
 127 def filter_units(line, units="imperial"):
 128     """Filter or convert units in a line of text between US/UK and metric."""
 129     import re
 130     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 131     # "X in. Hg (Y hPa)"
 132     dual_p = re.match(
 133         "(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 134         line
 135     )
 136     if dual_p:
 137         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 138         if units == "imperial": line = preamble + in_hg + trailer
 139         elif units == "metric": line = preamble + hpa + trailer
 140     # filter lines with both temperatures in the form of "X F (Y C)"
 141     dual_t = re.match(
 142         "(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 143         line
 144     )
 145     if dual_t:
 146         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 147         if units == "imperial": line = preamble + fahrenheit + trailer
 148         elif units == "metric": line = preamble + celsius + trailer
 149     # if metric is desired, convert distances in the form of "X mile(s)" to
 150     # "Y kilometer(s)"
 151     if units == "metric":
 152         imperial_d = re.match(
 153             "(.* )(\d+)( mile\(s\))(.*)",
 154             line
 155         )
 156         if imperial_d:
 157             preamble, mi, m_u, trailer = imperial_d.groups()
 158             line = preamble + str(int(round(int(mi)*1.609344))) \
 159                 + " kilometer(s)" + trailer
 160     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 161     # desired, convert to "Z KPH"
 162     imperial_s = re.match(
 163         "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 164         line
 165     )
 166     if imperial_s:
 167         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 168         if units == "imperial": line = preamble + mph + m_u + trailer
 169         elif units == "metric":
 170             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 171                 trailer
 172     imperial_s = re.match(
 173         "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 174         line
 175     )
 176     if imperial_s:
 177         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 178         if units == "imperial": line = preamble + mph + m_u + trailer
 179         elif units == "metric":
 180             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 181                 trailer
 182     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 183     # metric is desired, convert to "Y C"
 184     imperial_t = re.match(
 185         "(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 186         line
 187     )
 188     if imperial_t:
 189         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 190         if units == "imperial":
 191             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 192         elif units == "metric":
 193             line = preamble + parameter \
 194                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 195                 + trailer
 196     # hand off the resulting line
 197     return line
 198
 199 def get_uri(
 200     uri,
 201     ignore_fail=False,
 202     cache_data=False,
 203     cacheage=900,
 204     cachedir="."
 205 ):
 206     """Return a string containing the results of a URI GET."""
 207     if pyversion("3"):
 208         import urllib, urllib.error, urllib.request
 209         URLError = urllib.error.URLError
 210         urlopen = urllib.request.urlopen
 211     else:
 212         import urllib2 as urllib
 213         URLError = urllib.URLError
 214         urlopen = urllib.urlopen
 215     import os, time
 216     if cache_data:
 217         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 218         if not os.path.exists(dcachedir):
 219             try: os.makedirs(dcachedir)
 220             except (IOError, OSError): pass
 221         dcache_fn = os.path.join(
 222             dcachedir,
 223             uri.split(":",1)[1].replace("/","_")
 224         )
 225     now = time.time()
 226     if cache_data and os.access(dcache_fn, os.R_OK) \
 227         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 228         dcache_fd = open(dcache_fn)
 229         data = dcache_fd.read()
 230         dcache_fd.close()
 231     else:
 232         try:
 233             data = urlopen(uri).read().decode("utf-8")
 234         except URLError:
 235             if ignore_fail: return ""
 236             import os, sys
 237             sys.stderr.write("%s error: failed to retrieve\n   %s\n\n" % (
 238                 os.path.basename( sys.argv[0] ), uri))
 239             raise
 240         # Some data sources are HTML with the plain text wrapped in pre tags
 241         if "<pre>" in data:
 242             data = data[data.find("<pre>")+5:data.find("</pre>")]
 243         if cache_data:
 244             try:
 245                 import codecs
 246                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 247                 dcache_fd.write(data)
 248                 dcache_fd.close()
 249             except (IOError, OSError): pass
 250     return data
 251
 252 def get_metar(
 253     uri=None,
 254     verbose=False,
 255     quiet=False,
 256     headers=None,
 257     imperial=False,
 258     metric=False,
 259     cache_data=False,
 260     cacheage=900,
 261     cachedir="."
 262 ):
 263     """Return a summarized METAR for the specified station."""
 264     if not uri:
 265         import os, sys
 266         message = "%s error: METAR URI required for conditions\n" % \
 267             os.path.basename( sys.argv[0] )
 268         sys.stderr.write(message)
 269         sys.exit(1)
 270     metar = get_uri(
 271         uri,
 272         cache_data=cache_data,
 273         cacheage=cacheage,
 274         cachedir=cachedir
 275     )
 276     if pyversion("3") and type(metar) is bytes: metar = metar.decode("utf-8")
 277     if verbose: return metar
 278     else:
 279         import re
 280         lines = metar.split("\n")
 281         if not headers:
 282             headers = \
 283                 "relative_humidity," \
 284                 + "precipitation_last_hour," \
 285                 + "sky conditions," \
 286                 + "temperature," \
 287                 + "heat index," \
 288                 + "windchill," \
 289                 + "weather," \
 290                 + "wind"
 291         headerlist = headers.lower().replace("_"," ").split(",")
 292         output = []
 293         if not quiet:
 294             title = "Current conditions at %s"
 295             place = lines[0].split(", ")
 296             if len(place) > 1:
 297                 place = "%s, %s" % ( place[0].title(), place[1] )
 298             else: place = "<UNKNOWN>"
 299             output.append(title%place)
 300             output.append("Last updated " + lines[1])
 301         header_match = False
 302         for header in headerlist:
 303             for line in lines:
 304                 if line.lower().startswith(header + ":"):
 305                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 306                     if imperial: line = filter_units(line, units="imperial")
 307                     elif metric: line = filter_units(line, units="metric")
 308                     if quiet: output.append(line)
 309                     else: output.append("   " + line)
 310                     header_match = True
 311         if not header_match:
 312             output.append(
 313                 "(no conditions matched your header list, try with --verbose)"
 314             )
 315         return "\n".join(output)
 316
 317 def get_alert(
 318     uri=None,
 319     verbose=False,
 320     quiet=False,
 321     cache_data=False,
 322     cacheage=900,
 323     cachedir="."
 324 ):
 325     """Return alert notice for the specified URI."""
 326     if not uri:
 327         return ""
 328     alert = get_uri(
 329         uri,
 330         ignore_fail=True,
 331         cache_data=cache_data,
 332         cacheage=cacheage,
 333         cachedir=cachedir
 334     ).strip()
 335     if pyversion("3") and type(alert) is bytes: alert = alert.decode("utf-8")
 336     if alert:
 337         if verbose: return alert
 338         else:
 339             if alert.find("\nNATIONAL WEATHER SERVICE") == -1:
 340                 muted = False
 341             else:
 342                 muted = True
 343             lines = alert.split("\n")
 344             import time
 345             valid_time = time.strftime("%Y%m%d%H%M")
 346             output = []
 347             for line in lines:
 348                 if line.startswith("Expires:") \
 349                     and "Expires:" + valid_time > line:
 350                     return ""
 351                 if muted and line.startswith("NATIONAL WEATHER SERVICE"):
 352                     muted = False
 353                     line = ""
 354                 elif line == "&&":
 355                     line = ""
 356                 elif line == "$$":
 357                     muted = True
 358                 if line and not muted:
 359                     if quiet: output.append(line)
 360                     else: output.append("   " + line)
 361             return "\n".join(output)
 362
 363 def get_options(config):
 364     """Parse the options passed on the command line."""
 365
 366     # for optparse's builtin -h/--help option
 367     usage = \
 368         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 369
 370     # for optparse's builtin --version option
 371     verstring = "%prog " + weather_version
 372
 373     # create the parser
 374     import optparse
 375     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 376     # separate options object from list of arguments and return both
 377
 378     # the -a/--alert option
 379     if config.has_option("default", "alert"):
 380         default_alert = config.getboolean("default", "alert")
 381     else: default_alert = False
 382     option_parser.add_option("-a", "--alert",
 383         dest="alert",
 384         action="store_true",
 385         default=default_alert,
 386         help="include local alert notices")
 387
 388     # the --atypes option
 389     if config.has_option("default", "atypes"):
 390         default_atypes = config.get("default", "atypes")
 391     else:
 392         default_atypes = \
 393             "coastal_flood_statement," \
 394             + "flash_flood_statement," \
 395             + "flash_flood_warning," \
 396             + "flash_flood_watch," \
 397             + "flood_statement," \
 398             + "flood_warning," \
 399             + "severe_thunderstorm_warning," \
 400             + "severe_weather_statement," \
 401             + "special_weather_statement," \
 402             + "urgent_weather_message"
 403     option_parser.add_option("--atypes",
 404         dest="atypes",
 405         default=default_atypes,
 406         help="list of alert notification types to display")
 407
 408     # the --build-sets option
 409     option_parser.add_option("--build-sets",
 410         dest="build_sets",
 411         action="store_true",
 412         default=False,
 413         help="(re)build location correlation sets")
 414
 415     # the --cacheage option
 416     if config.has_option("default", "cacheage"):
 417         default_cacheage = config.getint("default", "cacheage")
 418     else: default_cacheage = 900
 419     option_parser.add_option("--cacheage",
 420         dest="cacheage",
 421         default=default_cacheage,
 422         help="duration in seconds to refresh cached data")
 423
 424     # the --cachedir option
 425     if config.has_option("default", "cachedir"):
 426         default_cachedir = config.get("default", "cachedir")
 427     else: default_cachedir = "~/.weather"
 428     option_parser.add_option("--cachedir",
 429         dest="cachedir",
 430         default=default_cachedir,
 431         help="directory for storing cached searches and data")
 432
 433     # the -f/--forecast option
 434     if config.has_option("default", "forecast"):
 435         default_forecast = config.getboolean("default", "forecast")
 436     else: default_forecast = False
 437     option_parser.add_option("-f", "--forecast",
 438         dest="forecast",
 439         action="store_true",
 440         default=default_forecast,
 441         help="include a local forecast")
 442
 443     # the --headers option
 444     if config.has_option("default", "headers"):
 445         default_headers = config.get("default", "headers")
 446     else:
 447         default_headers = \
 448             "temperature," \
 449             + "relative_humidity," \
 450             + "wind," \
 451             + "heat_index," \
 452             + "windchill," \
 453             + "weather," \
 454             + "sky_conditions," \
 455             + "precipitation_last_hour"
 456     option_parser.add_option("--headers",
 457         dest="headers",
 458         default=default_headers,
 459         help="list of conditions headers to display")
 460
 461     # the --imperial option
 462     if config.has_option("default", "imperial"):
 463         default_imperial = config.getboolean("default", "imperial")
 464     else: default_imperial = False
 465     option_parser.add_option("--imperial",
 466         dest="imperial",
 467         action="store_true",
 468         default=default_imperial,
 469         help="filter/convert conditions for US/UK units")
 470
 471     # the --info option
 472     option_parser.add_option("--info",
 473         dest="info",
 474         action="store_true",
 475         default=False,
 476         help="output detailed information for your search")
 477
 478     # the -l/--list option
 479     option_parser.add_option("-l", "--list",
 480         dest="list",
 481         action="store_true",
 482         default=False,
 483         help="list all configured aliases and cached searches")
 484
 485     # the --longlist option
 486     option_parser.add_option("--longlist",
 487         dest="longlist",
 488         action="store_true",
 489         default=False,
 490         help="display details of all configured aliases")
 491
 492     # the -m/--metric option
 493     if config.has_option("default", "metric"):
 494         default_metric = config.getboolean("default", "metric")
 495     else: default_metric = False
 496     option_parser.add_option("-m", "--metric",
 497         dest="metric",
 498         action="store_true",
 499         default=default_metric,
 500         help="filter/convert conditions for metric units")
 501
 502     # the -n/--no-conditions option
 503     if config.has_option("default", "conditions"):
 504         default_conditions = config.getboolean("default", "conditions")
 505     else: default_conditions = True
 506     option_parser.add_option("-n", "--no-conditions",
 507         dest="conditions",
 508         action="store_false",
 509         default=default_conditions,
 510         help="disable output of current conditions")
 511
 512     # the --no-cache option
 513     if config.has_option("default", "cache"):
 514         default_cache = config.getboolean("default", "cache")
 515     else: default_cache = True
 516     option_parser.add_option("--no-cache",
 517         dest="cache",
 518         action="store_false",
 519         default=True,
 520         help="disable all caching (searches and data)")
 521
 522     # the --no-cache-data option
 523     if config.has_option("default", "cache_data"):
 524         default_cache_data = config.getboolean("default", "cache_data")
 525     else: default_cache_data = True
 526     option_parser.add_option("--no-cache-data",
 527         dest="cache_data",
 528         action="store_false",
 529         default=True,
 530         help="disable retrieved data caching")
 531
 532     # the --no-cache-search option
 533     if config.has_option("default", "cache_search"):
 534         default_cache_search = config.getboolean("default", "cache_search")
 535     else: default_cache_search = True
 536     option_parser.add_option("--no-cache-search",
 537         dest="cache_search",
 538         action="store_false",
 539         default=True,
 540         help="disable search result caching")
 541
 542     # the -q/--quiet option
 543     if config.has_option("default", "quiet"):
 544         default_quiet = config.getboolean("default", "quiet")
 545     else: default_quiet = False
 546     option_parser.add_option("-q", "--quiet",
 547         dest="quiet",
 548         action="store_true",
 549         default=default_quiet,
 550         help="skip preambles and don't indent")
 551
 552     # the --setpath option
 553     if config.has_option("default", "setpath"):
 554         default_setpath = config.get("default", "setpath")
 555     else: default_setpath = ".:~/.weather"
 556     option_parser.add_option("--setpath",
 557         dest="setpath",
 558         default=default_setpath,
 559         help="directory search path for correlation sets")
 560
 561     # the -v/--verbose option
 562     if config.has_option("default", "verbose"):
 563         default_verbose = config.getboolean("default", "verbose")
 564     else: default_verbose = False
 565     option_parser.add_option("-v", "--verbose",
 566         dest="verbose",
 567         action="store_true",
 568         default=default_verbose,
 569         help="show full decoded feeds")
 570
 571     # deprecated options
 572     if config.has_option("default", "city"):
 573         default_city = config.get("default", "city")
 574     else: default_city = ""
 575     option_parser.add_option("-c", "--city",
 576         dest="city",
 577         default=default_city,
 578         help=optparse.SUPPRESS_HELP)
 579     if config.has_option("default", "id"):
 580         default_id = config.get("default", "id")
 581     else: default_id = ""
 582     option_parser.add_option("-i", "--id",
 583         dest="id",
 584         default=default_id,
 585         help=optparse.SUPPRESS_HELP)
 586     if config.has_option("default", "st"):
 587         default_st = config.get("default", "st")
 588     else: default_st = ""
 589     option_parser.add_option("-s", "--st",
 590         dest="st",
 591         default=default_st,
 592         help=optparse.SUPPRESS_HELP)
 593
 594     options, arguments = option_parser.parse_args()
 595     return options, arguments
 596
 597 def get_config():
 598     """Parse the aliases and configuration."""
 599     if pyversion("3"): import configparser
 600     else: import ConfigParser as configparser
 601     config = configparser.ConfigParser()
 602     import os
 603     rcfiles = [
 604         "/etc/weatherrc",
 605         "/etc/weather/weatherrc",
 606         os.path.expanduser("~/.weather/weatherrc"),
 607         os.path.expanduser("~/.weatherrc"),
 608         "weatherrc"
 609         ]
 610     for rcfile in rcfiles:
 611         if os.access(rcfile, os.R_OK):
 612             if pyversion("3"):
 613                 config.read(rcfile, encoding="utf-8")
 614             else:
 615                 config.read(rcfile)
 616     for section in config.sections():
 617         if section != section.lower():
 618             if config.has_section(section.lower()):
 619                 config.remove_section(section.lower())
 620             config.add_section(section.lower())
 621             for option,value in config.items(section):
 622                 config.set(section.lower(), option, value)
 623     return config
 624
 625 def integrate_search_cache(config, cachedir, setpath):
 626     """Add cached search results into the configuration."""
 627     if pyversion("3"): import configparser
 628     else: import ConfigParser as configparser
 629     import os, time
 630     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 631     if not os.access(scache_fn, os.R_OK): return config
 632     scache_fd = open(scache_fn)
 633     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 634     scache_fd.close()
 635     now = time.time()
 636     datafiles = data_index(setpath)
 637     if datafiles:
 638         data_freshness = sorted(
 639             [ x[1] for x in datafiles.values() ],
 640             reverse=True
 641         )[0]
 642     else: data_freshness = now
 643     if created < data_freshness <= now:
 644         try:
 645             os.remove(scache_fn)
 646             print( "[clearing outdated %s]" % scache_fn )
 647         except (IOError, OSError):
 648             pass
 649         return config
 650     scache = configparser.ConfigParser()
 651     if pyversion("3"):
 652         scache.read(scache_fn, encoding="utf-8")
 653     else:
 654         scache.read(scache_fn)
 655     for section in scache.sections():
 656         if not config.has_section(section):
 657             config.add_section(section)
 658             for option,value in scache.items(section):
 659                 config.set(section, option, value)
 660     return config
 661
 662 def list_aliases(config, detail=False):
 663     """Return a formatted list of aliases defined in the config."""
 664     if detail:
 665         output = "\n# configured alias details..."
 666         for section in sorted(config.sections()):
 667             output += "\n\n[%s]" % section
 668             for item in sorted(config.items(section)):
 669                 output += "\n%s = %s" % item
 670         output += "\n"
 671     else:
 672         output = "configured aliases and cached searches..."
 673         for section in sorted(config.sections()):
 674             if config.has_option(section, "description"):
 675                 description = config.get(section, "description")
 676             else: description = "(no description provided)"
 677             output += "\n   %s: %s" % (section, description)
 678     return output
 679
 680 def data_index(path):
 681     import os
 682     datafiles = {}
 683     for filename in ("airports", "places", "stations", "zctas", "zones"):
 684         for dirname in path.split(":"):
 685             for extension in ("", ".gz", ".txt"):
 686                 candidate = os.path.expanduser(
 687                     os.path.join( dirname, "".join( (filename, extension) ) )
 688                 )
 689                 if os.path.exists(candidate):
 690                     datafiles[filename] = (
 691                         candidate,
 692                         os.stat(candidate).st_mtime
 693                     )
 694                     break
 695             if filename in datafiles:
 696                 break
 697     return datafiles
 698
 699 def guess(
 700     expression,
 701     path=".",
 702     max_results=20,
 703     info=False,
 704     cache_search=False,
 705     cacheage=900,
 706     cachedir=".",
 707     quiet=False
 708 ):
 709     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 710     import codecs, datetime, time, os, re, sys
 711     if pyversion("3"): import configparser
 712     else: import ConfigParser as configparser
 713     datafiles = data_index(path)
 714     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 715     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 716     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 717     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 718     elif re.match(
 719         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 720         expression
 721     ):
 722         searchtype = "coordinates"
 723     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 724     else:
 725         searchtype = "name"
 726         cache_search = False
 727     if cache_search: action = "caching"
 728     else: action = "using"
 729     if info:
 730         scores = [
 731             (0.005, "bad"),
 732             (0.025, "poor"),
 733             (0.160, "suspect"),
 734             (0.500, "mediocre"),
 735             (0.840, "good"),
 736             (0.975, "great"),
 737             (0.995, "excellent"),
 738             (1.000, "ideal"),
 739         ]
 740     if not quiet: print("Searching via %s..."%searchtype)
 741     stations = configparser.ConfigParser()
 742     dataname = "stations"
 743     if dataname in datafiles:
 744         datafile = datafiles[dataname][0]
 745         if datafile.endswith(".gz"):
 746             import gzip
 747             if pyversion("3"):
 748                 stations.read_string(
 749                     gzip.open(datafile).read().decode("utf-8") )
 750             else: stations.readfp( gzip.open(datafile) )
 751         else:
 752             if pyversion("3"):
 753                 stations.read(datafile, encoding="utf-8")
 754             else:
 755                 stations.read(datafile)
 756     else:
 757         message = "%s error: can't find \"%s\" data file\n" % (
 758             os.path.basename( sys.argv[0] ),
 759             dataname
 760         )
 761         sys.stderr.write(message)
 762         exit(1)
 763     zones = configparser.ConfigParser()
 764     dataname = "zones"
 765     if dataname in datafiles:
 766         datafile = datafiles[dataname][0]
 767         if datafile.endswith(".gz"):
 768             import gzip
 769             if pyversion("3"):
 770                 zones.read_string( gzip.open(datafile).read().decode("utf-8") )
 771             else: zones.readfp( gzip.open(datafile) )
 772         else:
 773             if pyversion("3"):
 774                 zones.read(datafile, encoding="utf-8")
 775             else:
 776                 zones.read(datafile)
 777     else:
 778         message = "%s error: can't find \"%s\" data file\n" % (
 779             os.path.basename( sys.argv[0] ),
 780             dataname
 781         )
 782         sys.stderr.write(message)
 783         exit(1)
 784     search = None
 785     station = ("", 0)
 786     zone = ("", 0)
 787     dataset = None
 788     possibilities = []
 789     uris = {}
 790     if searchtype == "airport":
 791         expression = expression.lower()
 792         airports = configparser.ConfigParser()
 793         dataname = "airports"
 794         if dataname in datafiles:
 795             datafile = datafiles[dataname][0]
 796             if datafile.endswith(".gz"):
 797                 import gzip
 798                 if pyversion("3"):
 799                     airports.read_string(
 800                         gzip.open(datafile).read().decode("utf-8") )
 801                 else: airports.readfp( gzip.open(datafile) )
 802             else:
 803                 if pyversion("3"):
 804                     airports.read(datafile, encoding="utf-8")
 805                 else:
 806                     airports.read(datafile)
 807         else:
 808             message = "%s error: can't find \"%s\" data file\n" % (
 809                 os.path.basename( sys.argv[0] ),
 810                 dataname
 811             )
 812             sys.stderr.write(message)
 813             exit(1)
 814         if airports.has_section(expression) \
 815             and airports.has_option(expression, "station"):
 816             search = (expression, "IATA/FAA airport code %s" % expression)
 817             station = ( airports.get(expression, "station"), 0 )
 818             if stations.has_option(station[0], "zone"):
 819                 zone = eval( stations.get(station[0], "zone") )
 820                 dataset = stations
 821             if not ( info or quiet ) \
 822                 and stations.has_option( station[0], "description" ):
 823                 print(
 824                     "[%s result %s]" % (
 825                         action,
 826                         stations.get(station[0], "description")
 827                     )
 828                 )
 829         else:
 830             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 831                 expression,
 832                 datafiles["airports"][0]
 833             )
 834             sys.stderr.write(message)
 835             exit(1)
 836     elif searchtype == "station":
 837         expression = expression.lower()
 838         if stations.has_section(expression):
 839             station = (expression, 0)
 840             if not search:
 841                 search = (expression, "ICAO station code %s" % expression)
 842             if stations.has_option(expression, "zone"):
 843                 zone = eval( stations.get(expression, "zone") )
 844                 dataset = stations
 845             if not ( info or quiet ) \
 846                 and stations.has_option(expression, "description"):
 847                 print(
 848                     "[%s result %s]" % (
 849                         action,
 850                         stations.get(expression, "description")
 851                     )
 852                 )
 853         else:
 854             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 855                 expression,
 856                 datafiles["stations"][0]
 857             )
 858             sys.stderr.write(message)
 859             exit(1)
 860     elif searchtype == "zone":
 861         expression = expression.lower()
 862         if zones.has_section(expression) \
 863             and zones.has_option(expression, "station"):
 864             zone = (expression, 0)
 865             station = eval( zones.get(expression, "station") )
 866             dataset = zones
 867             search = (expression, "NWS/NOAA weather zone %s" % expression)
 868             if not ( info or quiet ) \
 869                 and zones.has_option(expression, "description"):
 870                 print(
 871                     "[%s result %s]" % (
 872                         action,
 873                         zones.get(expression, "description")
 874                     )
 875                 )
 876         else:
 877             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 878                 expression,
 879                 datafiles["zones"][0]
 880             )
 881             sys.stderr.write(message)
 882             exit(1)
 883     elif searchtype == "ZCTA":
 884         zctas = configparser.ConfigParser()
 885         dataname = "zctas"
 886         if dataname in datafiles:
 887             datafile = datafiles[dataname][0]
 888             if datafile.endswith(".gz"):
 889                 import gzip
 890                 if pyversion("3"):
 891                     zctas.read_string(
 892                         gzip.open(datafile).read().decode("utf-8") )
 893                 else: zctas.readfp( gzip.open(datafile) )
 894             else:
 895                 if pyversion("3"):
 896                     zctas.read(datafile, encoding="utf-8")
 897                 else:
 898                     zctas.read(datafile)
 899         else:
 900             message = "%s error: can't find \"%s\" data file\n" % (
 901                 os.path.basename( sys.argv[0] ),
 902                 dataname
 903             )
 904             sys.stderr.write(message)
 905             exit(1)
 906         dataset = zctas
 907         if zctas.has_section(expression) \
 908             and zctas.has_option(expression, "station"):
 909             station = eval( zctas.get(expression, "station") )
 910             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 911             if zctas.has_option(expression, "zone"):
 912                 zone = eval( zctas.get(expression, "zone") )
 913         else:
 914             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 915                 expression,
 916                 datafiles["zctas"][0]
 917             )
 918             sys.stderr.write(message)
 919             exit(1)
 920     elif searchtype == "coordinates":
 921         search = (expression, "Geographic coordinates %s" % expression)
 922         stationtable = {}
 923         for station in stations.sections():
 924             if stations.has_option(station, "location"):
 925                 stationtable[station] = {
 926                     "location": eval( stations.get(station, "location") )
 927                 }
 928         station = closest( gecos(expression), stationtable, "location", 0.1 )
 929         if not station[0]:
 930             message = "No ICAO weather station found near %s.\n" % expression
 931             sys.stderr.write(message)
 932             exit(1)
 933         zonetable = {}
 934         for zone in zones.sections():
 935             if zones.has_option(zone, "centroid"):
 936                 zonetable[zone] = {
 937                     "centroid": eval( zones.get(zone, "centroid") )
 938                 }
 939         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 940         if not zone[0]:
 941             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 942                 % expression
 943             sys.stderr.write(message)
 944     elif searchtype in ("FIPS", "name"):
 945         places = configparser.ConfigParser()
 946         dataname = "places"
 947         if dataname in datafiles:
 948             datafile = datafiles[dataname][0]
 949             if datafile.endswith(".gz"):
 950                 import gzip
 951                 if pyversion("3"):
 952                     places.read_string(
 953                         gzip.open(datafile).read().decode("utf-8") )
 954                 else: places.readfp( gzip.open(datafile) )
 955             else:
 956                 if pyversion("3"):
 957                     places.read(datafile, encoding="utf-8")
 958                 else:
 959                     places.read(datafile)
 960         else:
 961             message = "%s error: can't find \"%s\" data file\n" % (
 962                 os.path.basename( sys.argv[0] ),
 963                 dataname
 964             )
 965             sys.stderr.write(message)
 966             exit(1)
 967         dataset = places
 968         place = expression.lower()
 969         if places.has_section(place) and places.has_option(place, "station"):
 970             station = eval( places.get(place, "station") )
 971             search = (expression, "Census Place %s" % expression)
 972             if places.has_option(place, "description"):
 973                 search = (
 974                     search[0],
 975                     search[1] + ", %s" % places.get(place, "description")
 976                 )
 977             if places.has_option(place, "zone"):
 978                 zone = eval( places.get(place, "zone") )
 979             if not ( info or quiet ) \
 980                 and places.has_option(place, "description"):
 981                 print(
 982                     "[%s result %s]" % (
 983                         action,
 984                         places.get(place, "description")
 985                     )
 986                 )
 987         else:
 988             for place in places.sections():
 989                 if places.has_option(place, "description") \
 990                     and places.has_option(place, "station") \
 991                     and re.search(
 992                         expression,
 993                         places.get(place, "description"),
 994                         re.I
 995                     ):
 996                         possibilities.append(place)
 997             for place in stations.sections():
 998                 if stations.has_option(place, "description") \
 999                     and re.search(
1000                         expression,
1001                         stations.get(place, "description"),
1002                         re.I
1003                     ):
1004                         possibilities.append(place)
1005             for place in zones.sections():
1006                 if zones.has_option(place, "description") \
1007                     and zones.has_option(place, "station") \
1008                     and re.search(
1009                         expression,
1010                         zones.get(place, "description"),
1011                         re.I
1012                     ):
1013                         possibilities.append(place)
1014             if len(possibilities) == 1:
1015                 place = possibilities[0]
1016                 if places.has_section(place):
1017                     station = eval( places.get(place, "station") )
1018                     description = places.get(place, "description")
1019                     if places.has_option(place, "zone"):
1020                         zone = eval( places.get(place, "zone" ) )
1021                     search = ( expression, "%s: %s" % (place, description) )
1022                 elif stations.has_section(place):
1023                     station = (place, 0.0)
1024                     description = stations.get(place, "description")
1025                     if stations.has_option(place, "zone"):
1026                         zone = eval( stations.get(place, "zone" ) )
1027                     search = ( expression, "ICAO station code %s" % place )
1028                 elif zones.has_section(place):
1029                     station = eval( zones.get(place, "station") )
1030                     description = zones.get(place, "description")
1031                     zone = (place, 0.0)
1032                     search = ( expression, "NWS/NOAA weather zone %s" % place )
1033                 if not ( info or quiet ):
1034                     print( "[%s result %s]" % (action, description) )
1035             if not possibilities and not station[0]:
1036                 message = "No FIPS code/census area match in the %s file.\n" % (
1037                     datafiles["places"][0]
1038                 )
1039                 sys.stderr.write(message)
1040                 exit(1)
1041     if station[0]:
1042         uris["metar"] = stations.get( station[0], "metar" )
1043         if zone[0]:
1044             for key,value in zones.items( zone[0] ):
1045                 if key not in ("centroid", "description", "station"):
1046                     uris[key] = value
1047     elif possibilities:
1048         count = len(possibilities)
1049         if count <= max_results:
1050             print( "Your search is ambiguous, returning %s matches:" % count )
1051             for place in sorted(possibilities):
1052                 if places.has_section(place):
1053                     print(
1054                         "   [%s] %s" % (
1055                             place,
1056                             places.get(place, "description")
1057                         )
1058                     )
1059                 elif stations.has_section(place):
1060                     print(
1061                         "   [%s] %s" % (
1062                             place,
1063                             stations.get(place, "description")
1064                         )
1065                     )
1066                 elif zones.has_section(place):
1067                     print(
1068                         "   [%s] %s" % (
1069                             place,
1070                             zones.get(place, "description")
1071                         )
1072                     )
1073         else:
1074             print(
1075                 "Your search is too ambiguous, returning %s matches." % count
1076             )
1077         exit(0)
1078     if info:
1079         stationlist = []
1080         zonelist = []
1081         if dataset:
1082             for section in dataset.sections():
1083                 if dataset.has_option(section, "station"):
1084                     stationlist.append(
1085                         eval( dataset.get(section, "station") )[1]
1086                     )
1087                 if dataset.has_option(section, "zone"):
1088                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1089         stationlist.sort()
1090         zonelist.sort()
1091         scount = len(stationlist)
1092         zcount = len(zonelist)
1093         sranks = []
1094         zranks = []
1095         for score in scores:
1096             if stationlist:
1097                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1098             if zonelist:
1099                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1100         description = search[1]
1101         uris["description"] = description
1102         print(
1103             "%s\n%s" % ( description, "-" * len(description) )
1104         )
1105         print(
1106             "%s: %s" % (
1107                 station[0],
1108                 stations.get( station[0], "description" )
1109             )
1110         )
1111         km = radian_to_km*station[1]
1112         mi = radian_to_mi*station[1]
1113         if sranks and not description.startswith("ICAO station code "):
1114             for index in range(0, len(scores)):
1115                 if station[1] >= sranks[index]:
1116                     score = scores[index][1]
1117                     break
1118             print(
1119                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1120             )
1121         elif searchtype == "coordinates":
1122             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1123         if zone[0]:
1124             print(
1125                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1126             )
1127         km = radian_to_km*zone[1]
1128         mi = radian_to_mi*zone[1]
1129         if zranks and not description.startswith("NWS/NOAA weather zone "):
1130             for index in range(0, len(scores)):
1131                 if zone[1] >= zranks[index]:
1132                     score = scores[index][1]
1133                     break
1134             print(
1135                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1136             )
1137         elif searchtype == "coordinates" and zone[0]:
1138             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1139     if cache_search:
1140         now = time.time()
1141         nowstamp = "%s (%s)" % (
1142             now,
1143             datetime.datetime.isoformat(
1144                 datetime.datetime.fromtimestamp(now),
1145                 " "
1146             )
1147         )
1148         search_cache = ["\n"]
1149         search_cache.append( "[%s]\n" % search[0] )
1150         search_cache.append( "cached = %s\n" % nowstamp )
1151         for uriname in sorted(uris.keys()):
1152             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1153         real_cachedir = os.path.expanduser(cachedir)
1154         if not os.path.exists(real_cachedir):
1155             try: os.makedirs(real_cachedir)
1156             except (IOError, OSError): pass
1157         scache_fn = os.path.join(real_cachedir, "searches")
1158         if not os.path.exists(scache_fn):
1159             then = sorted(
1160                     [ x[1] for x in datafiles.values() ],
1161                     reverse=True
1162                 )[0]
1163             thenstamp = "%s (%s)" % (
1164                 then,
1165                 datetime.datetime.isoformat(
1166                     datetime.datetime.fromtimestamp(then),
1167                     " "
1168                 )
1169             )
1170             search_cache.insert(
1171                 0,
1172                 "# based on data files from: %s\n" % thenstamp
1173             )
1174         try:
1175             scache_existing = configparser.ConfigParser()
1176             if pyversion("3"):
1177                 scache_existing.read(scache_fn, encoding="utf-8")
1178             else:
1179                 scache_existing.read(scache_fn)
1180             if not scache_existing.has_section(search[0]):
1181                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1182                 scache_fd.writelines(search_cache)
1183                 scache_fd.close()
1184         except (IOError, OSError): pass
1185     if not info:
1186         return(uris)
1187
1188 def closest(position, nodes, fieldname, angle=None):
1189     import math
1190     if not angle: angle = 2*math.pi
1191     match = None
1192     for name in nodes:
1193         if fieldname in nodes[name]:
1194             node = nodes[name][fieldname]
1195             if node and abs( position[0]-node[0] ) < angle:
1196                 if abs( position[1]-node[1] ) < angle \
1197                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1198                     if position == node:
1199                         angle = 0
1200                         match = name
1201                     else:
1202                         candidate = math.acos(
1203                             math.sin( position[0] ) * math.sin( node[0] ) \
1204                                 + math.cos( position[0] ) \
1205                                 * math.cos( node[0] ) \
1206                                 * math.cos( position[1] - node[1] )
1207                             )
1208                         if candidate < angle:
1209                             angle = candidate
1210                             match = name
1211     if match: match = str(match)
1212     return (match, angle)
1213
1214 def gecos(formatted):
1215     import math, re
1216     coordinates = formatted.split(",")
1217     for coordinate in range(0, 2):
1218         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1219             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1220             coordinates[coordinate].strip().lower()
1221         ).groups()
1222         value = float(degrees)
1223         if minutes: value += float(minutes)/60
1224         if seconds: value += float(seconds)/3600
1225         if hemisphere and hemisphere in "sw": value *= -1
1226         coordinates[coordinate] = math.radians(value)
1227     return tuple(coordinates)
1228
1229 def correlate():
1230     import codecs, csv, datetime, hashlib, os, re, sys, time, zipfile
1231     if pyversion("3"): import configparser
1232     else: import ConfigParser as configparser
1233     for filename in os.listdir("."):
1234         if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename):
1235             gcounties_an = filename
1236             gcounties_fn = filename[:-4] + ".txt"
1237         elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename):
1238             gcousubs_an = filename
1239             gcousubs_fn = filename[:-4] + ".txt"
1240         elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename):
1241             gplace_an = filename
1242             gplace_fn = filename[:-4] + ".txt"
1243         elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename):
1244             gzcta_an = filename
1245             gzcta_fn = filename[:-4] + ".txt"
1246         elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename):
1247             cpfzcf_fn = filename
1248     nsdcccc_fn = "nsd_cccc.txt"
1249     ourairports_fn = "airports.csv"
1250     overrides_fn = "overrides.conf"
1251     overrideslog_fn = "overrides.log"
1252     slist_fn = "slist"
1253     zlist_fn = "zlist"
1254     qalog_fn = "qa.log"
1255     airports_fn = "airports"
1256     places_fn = "places"
1257     stations_fn = "stations"
1258     zctas_fn = "zctas"
1259     zones_fn = "zones"
1260     header = """\
1261 %s
1262 # generated by %s on %s from these public domain sources:
1263 #
1264 # https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html
1265 # %s %s %s
1266 # %s %s %s
1267 # %s %s %s
1268 # %s %s %s
1269 #
1270 # https://www.weather.gov/gis/ZoneCounty/
1271 # %s %s %s
1272 #
1273 # https://tgftp.nws.noaa.gov/data/
1274 # %s %s %s
1275 #
1276 # https://ourairports.com/data/
1277 # %s %s %s
1278 #
1279 # ...and these manually-generated or hand-compiled adjustments:
1280 # %s %s %s
1281 # %s %s %s
1282 # %s %s %s\
1283 """ % (
1284         weather_copyright,
1285         os.path.basename( sys.argv[0] ),
1286         datetime.date.isoformat(
1287             datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
1288         ),
1289         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1290         datetime.date.isoformat(
1291             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
1292         ),
1293         gcounties_an,
1294         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1295         datetime.date.isoformat(
1296             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
1297         ),
1298         gcousubs_an,
1299         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
1300         datetime.date.isoformat(
1301             datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
1302         ),
1303         gplace_an,
1304         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1305         datetime.date.isoformat(
1306             datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
1307         ),
1308         gzcta_an,
1309         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1310         datetime.date.isoformat(
1311             datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
1312         ),
1313         cpfzcf_fn,
1314         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1315         datetime.date.isoformat(
1316             datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
1317         ),
1318         nsdcccc_fn,
1319         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
1320         datetime.date.isoformat(
1321             datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
1322         ),
1323         ourairports_fn,
1324         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1325         datetime.date.isoformat(
1326             datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
1327         ),
1328         overrides_fn,
1329         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1330         datetime.date.isoformat(
1331             datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
1332         ),
1333         slist_fn,
1334         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1335         datetime.date.isoformat(
1336             datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
1337         ),
1338         zlist_fn
1339     )
1340     airports = {}
1341     places = {}
1342     stations = {}
1343     zctas = {}
1344     zones = {}
1345     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1346     sys.stdout.write(message)
1347     sys.stdout.flush()
1348     count = 0
1349     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
1350     columns = gcounties.readline().decode("utf-8").strip().split("\t")
1351     for line in gcounties:
1352         fields = line.decode("utf-8").strip().split("\t")
1353         f_geoid = fields[ columns.index("GEOID") ].strip()
1354         f_name = fields[ columns.index("NAME") ].strip()
1355         f_usps = fields[ columns.index("USPS") ].strip()
1356         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1357         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1358         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1359             fips = "fips%s" % f_geoid
1360             if fips not in places: places[fips] = {}
1361             places[fips]["centroid"] = gecos(
1362                 "%s,%s" % (f_intptlat, f_intptlong)
1363             )
1364             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1365             count += 1
1366     gcounties.close()
1367     print("done (%s lines)." % count)
1368     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1369     sys.stdout.write(message)
1370     sys.stdout.flush()
1371     count = 0
1372     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
1373     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
1374     for line in gcousubs:
1375         fields = line.decode("utf-8").strip().split("\t")
1376         f_geoid = fields[ columns.index("GEOID") ].strip()
1377         f_name = fields[ columns.index("NAME") ].strip()
1378         f_usps = fields[ columns.index("USPS") ].strip()
1379         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1380         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1381         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1382             fips = "fips%s" % f_geoid
1383             if fips not in places: places[fips] = {}
1384             places[fips]["centroid"] = gecos(
1385                 "%s,%s" % (f_intptlat, f_intptlong)
1386             )
1387             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1388             count += 1
1389     gcousubs.close()
1390     print("done (%s lines)." % count)
1391     message = "Reading %s:%s..." % (gplace_an, gplace_fn)
1392     sys.stdout.write(message)
1393     sys.stdout.flush()
1394     count = 0
1395     gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
1396     columns = gplace.readline().decode("utf-8").strip().split("\t")
1397     for line in gplace:
1398         fields = line.decode("utf-8").strip().split("\t")
1399         f_geoid = fields[ columns.index("GEOID") ].strip()
1400         f_name = fields[ columns.index("NAME") ].strip()
1401         f_usps = fields[ columns.index("USPS") ].strip()
1402         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1403         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1404         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1405             fips = "fips%s" % f_geoid
1406             if fips not in places: places[fips] = {}
1407             places[fips]["centroid"] = gecos(
1408                 "%s,%s" % (f_intptlat, f_intptlong)
1409             )
1410             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1411             count += 1
1412     gplace.close()
1413     print("done (%s lines)." % count)
1414     message = "Reading %s..." % slist_fn
1415     sys.stdout.write(message)
1416     sys.stdout.flush()
1417     count = 0
1418     slist = codecs.open(slist_fn, "r", "utf-8")
1419     for line in slist:
1420         icao = line.split("#")[0].strip()
1421         if icao:
1422             stations[icao] = {
1423                 "metar": "https://tgftp.nws.noaa.gov/data/observations/"\
1424                     + "metar/decoded/%s.TXT" % icao.upper()
1425             }
1426             count += 1
1427     slist.close()
1428     print("done (%s lines)." % count)
1429     message = "Reading %s..." % nsdcccc_fn
1430     sys.stdout.write(message)
1431     sys.stdout.flush()
1432     count = 0
1433     nsdcccc = codecs.open(nsdcccc_fn, "r", "utf-8")
1434     for line in nsdcccc:
1435         line = str(line)
1436         fields = line.split(";")
1437         icao = fields[0].strip().lower()
1438         if icao in stations:
1439             description = []
1440             name = " ".join( fields[3].strip().title().split() )
1441             if name: description.append(name)
1442             st = fields[4].strip()
1443             if st: description.append(st)
1444             country = " ".join( fields[5].strip().title().split() )
1445             if country: description.append(country)
1446             if description:
1447                 stations[icao]["description"] = ", ".join(description)
1448             lat, lon = fields[7:9]
1449             if lat and lon:
1450                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1451             elif "location" not in stations[icao]:
1452                 lat, lon = fields[5:7]
1453                 if lat and lon:
1454                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1455         count += 1
1456     nsdcccc.close()
1457     print("done (%s lines)." % count)
1458     message = "Reading %s..." % ourairports_fn
1459     sys.stdout.write(message)
1460     sys.stdout.flush()
1461     count = 0
1462     ourairports = open(ourairports_fn, "r")
1463     for row in csv.reader(ourairports):
1464         icao = row[12].lower()
1465         if icao in stations:
1466             iata = row[13].lower()
1467             if len(iata) == 3: airports[iata] = { "station": icao }
1468             if "description" not in stations[icao]:
1469                 description = []
1470                 name = row[3]
1471                 if name: description.append(name)
1472                 municipality = row[10]
1473                 if municipality: description.append(municipality)
1474                 region = row[9]
1475                 country = row[8]
1476                 if region:
1477                     if "-" in region:
1478                         c,r = region.split("-", 1)
1479                         if c == country: region = r
1480                     description.append(region)
1481                 if country:
1482                     description.append(country)
1483                 if description:
1484                     stations[icao]["description"] = ", ".join(description)
1485             if "location" not in stations[icao]:
1486                 lat = row[4]
1487                 if lat:
1488                     lon = row[5]
1489                     if lon:
1490                         stations[icao]["location"] = gecos(
1491                             "%s,%s" % (lat, lon)
1492                         )
1493         count += 1
1494     ourairports.close()
1495     print("done (%s lines)." % count)
1496     message = "Reading %s..." % zlist_fn
1497     sys.stdout.write(message)
1498     sys.stdout.flush()
1499     count = 0
1500     zlist = codecs.open(zlist_fn, "r", "utf-8")
1501     for line in zlist:
1502         line = line.split("#")[0].strip()
1503         if line:
1504             zones[line] = {}
1505             count += 1
1506     zlist.close()
1507     print("done (%s lines)." % count)
1508     message = "Reading %s..." % cpfzcf_fn
1509     sys.stdout.write(message)
1510     sys.stdout.flush()
1511     count = 0
1512     cpfz = {}
1513     cpfzcf = codecs.open(cpfzcf_fn, "r", "utf-8")
1514     for line in cpfzcf:
1515         fields = line.strip().split("|")
1516         if len(fields) == 11 \
1517             and fields[0] and fields[1] and fields[9] and fields[10]:
1518             zone = "z".join( fields[:2] ).lower()
1519             if zone in zones:
1520                 state = fields[0]
1521                 if state:
1522                     zones[zone]["coastal_flood_statement"] = (
1523                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1524                         "flood/coastal/%s/%s.txt" % (state.lower(), zone))
1525                     zones[zone]["flash_flood_statement"] = (
1526                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1527                         "flash_flood/statement/%s/%s.txt"
1528                         % (state.lower(), zone))
1529                     zones[zone]["flash_flood_warning"] = (
1530                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1531                         "flash_flood/warning/%s/%s.txt"
1532                         % (state.lower(), zone))
1533                     zones[zone]["flash_flood_watch"] = (
1534                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1535                         "flash_flood/watch/%s/%s.txt" % (state.lower(), zone))
1536                     zones[zone]["flood_statement"] = (
1537                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1538                         "flood/statement/%s/%s.txt" % (state.lower(), zone))
1539                     zones[zone]["flood_warning"] = (
1540                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1541                         "flood/warning/%s/%s.txt" % (state.lower(), zone))
1542                     zones[zone]["severe_thunderstorm_warning"] = (
1543                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1544                         "thunderstorm/%s/%s.txt" % (state.lower(), zone))
1545                     zones[zone]["severe_weather_statement"] = (
1546                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1547                         "severe_weather_stmt/%s/%s.txt"
1548                         % (state.lower(), zone))
1549                     zones[zone]["short_term_forecast"] = (
1550                         "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/"
1551                         "%s/%s.txt" % (state.lower(), zone))
1552                     zones[zone]["special_weather_statement"] = (
1553                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1554                         "special_weather_stmt/%s/%s.txt"
1555                         % (state.lower(), zone))
1556                     zones[zone]["state_forecast"] = (
1557                         "https://tgftp.nws.noaa.gov/data/forecasts/state/"
1558                         "%s/%s.txt" % (state.lower(), zone))
1559                     zones[zone]["urgent_weather_message"] = (
1560                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1561                         "non_precip/%s/%s.txt" % (state.lower(), zone))
1562                     zones[zone]["zone_forecast"] = (
1563                         "https://tgftp.nws.noaa.gov/data/forecasts/zone/"
1564                         "%s/%s.txt" % (state.lower(), zone))
1565                 description = fields[3].strip()
1566                 fips = "fips%s"%fields[6]
1567                 county = fields[5]
1568                 if county:
1569                     if description.endswith(county):
1570                         description += " County"
1571                     else:
1572                         description += ", %s County" % county
1573                 description += ", %s, US" % state
1574                 zones[zone]["description"] = description
1575                 zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
1576                 if fips in places and not zones[zone]["centroid"]:
1577                     zones[zone]["centroid"] = places[fips]["centroid"]
1578         count += 1
1579     cpfzcf.close()
1580     print("done (%s lines)." % count)
1581     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1582     sys.stdout.write(message)
1583     sys.stdout.flush()
1584     count = 0
1585     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
1586     columns = gzcta.readline().decode("utf-8").strip().split("\t")
1587     for line in gzcta:
1588         fields = line.decode("utf-8").strip().split("\t")
1589         f_geoid = fields[ columns.index("GEOID") ].strip()
1590         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1591         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1592         if f_geoid and f_intptlat and f_intptlong:
1593             if f_geoid not in zctas: zctas[f_geoid] = {}
1594             zctas[f_geoid]["centroid"] = gecos(
1595                 "%s,%s" % (f_intptlat, f_intptlong)
1596             )
1597             count += 1
1598     gzcta.close()
1599     print("done (%s lines)." % count)
1600     message = "Reading %s..." % overrides_fn
1601     sys.stdout.write(message)
1602     sys.stdout.flush()
1603     count = 0
1604     added = 0
1605     removed = 0
1606     changed = 0
1607     overrides = configparser.ConfigParser()
1608     overrides.readfp( codecs.open(overrides_fn, "r", "utf8") )
1609     overrideslog = []
1610     for section in overrides.sections():
1611         addopt = 0
1612         chgopt = 0
1613         if section.startswith("-"):
1614             section = section[1:]
1615             delete = True
1616         else: delete = False
1617         if re.match("[A-Za-z]{3}$", section):
1618             if delete:
1619                 if section in airports:
1620                     del( airports[section] )
1621                     logact = "removed airport %s" % section
1622                     removed += 1
1623                 else:
1624                     logact = "tried to remove nonexistent airport %s" % section
1625             else:
1626                 if section in airports:
1627                     logact = "changed airport %s" % section
1628                     changed += 1
1629                 else:
1630                     airports[section] = {}
1631                     logact = "added airport %s" % section
1632                     added += 1
1633                 for key,value in overrides.items(section):
1634                     if key in airports[section]: chgopt += 1
1635                     else: addopt += 1
1636                     if key in ("centroid", "location"):
1637                         airports[section][key] = eval(value)
1638                     else:
1639                         airports[section][key] = value
1640                 if addopt and chgopt:
1641                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1642                 elif addopt: logact += " (+%s options)" % addopt
1643                 elif chgopt: logact += " (!%s options)" % chgopt
1644         elif re.match("[A-Za-z0-9]{4}$", section):
1645             if delete:
1646                 if section in stations:
1647                     del( stations[section] )
1648                     logact = "removed station %s" % section
1649                     removed += 1
1650                 else:
1651                     logact = "tried to remove nonexistent station %s" % section
1652             else:
1653                 if section in stations:
1654                     logact = "changed station %s" % section
1655                     changed += 1
1656                 else:
1657                     stations[section] = {}
1658                     logact = "added station %s" % section
1659                     added += 1
1660                 for key,value in overrides.items(section):
1661                     if key in stations[section]: chgopt += 1
1662                     else: addopt += 1
1663                     if key in ("centroid", "location"):
1664                         stations[section][key] = eval(value)
1665                     else:
1666                         stations[section][key] = value
1667                 if addopt and chgopt:
1668                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1669                 elif addopt: logact += " (+%s options)" % addopt
1670                 elif chgopt: logact += " (!%s options)" % chgopt
1671         elif re.match("[0-9]{5}$", section):
1672             if delete:
1673                 if section in zctas:
1674                     del( zctas[section] )
1675                     logact = "removed zcta %s" % section
1676                     removed += 1
1677                 else:
1678                     logact = "tried to remove nonexistent zcta %s" % section
1679             else:
1680                 if section in zctas:
1681                     logact = "changed zcta %s" % section
1682                     changed += 1
1683                 else:
1684                     zctas[section] = {}
1685                     logact = "added zcta %s" % section
1686                     added += 1
1687                 for key,value in overrides.items(section):
1688                     if key in zctas[section]: chgopt += 1
1689                     else: addopt += 1
1690                     if key in ("centroid", "location"):
1691                         zctas[section][key] = eval(value)
1692                     else:
1693                         zctas[section][key] = value
1694                 if addopt and chgopt:
1695                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1696                 elif addopt: logact += " (+%s options)" % addopt
1697                 elif chgopt: logact += " (!%s options)" % chgopt
1698         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1699             if delete:
1700                 if section in zones:
1701                     del( zones[section] )
1702                     logact = "removed zone %s" % section
1703                     removed += 1
1704                 else:
1705                     logact = "tried to remove nonexistent zone %s" % section
1706             else:
1707                 if section in zones:
1708                     logact = "changed zone %s" % section
1709                     changed += 1
1710                 else:
1711                     zones[section] = {}
1712                     logact = "added zone %s" % section
1713                     added += 1
1714                 for key,value in overrides.items(section):
1715                     if key in zones[section]: chgopt += 1
1716                     else: addopt += 1
1717                     if key in ("centroid", "location"):
1718                         zones[section][key] = eval(value)
1719                     else:
1720                         zones[section][key] = value
1721                 if addopt and chgopt:
1722                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1723                 elif addopt: logact += " (+%s options)" % addopt
1724                 elif chgopt: logact += " (!%s options)" % chgopt
1725         elif re.match("fips[0-9]+$", section):
1726             if delete:
1727                 if section in places:
1728                     del( places[section] )
1729                     logact = "removed place %s" % section
1730                     removed += 1
1731                 else:
1732                     logact = "tried to remove nonexistent place %s" % section
1733             else:
1734                 if section in places:
1735                     logact = "changed place %s" % section
1736                     changed += 1
1737                 else:
1738                     places[section] = {}
1739                     logact = "added place %s" % section
1740                     added += 1
1741                 for key,value in overrides.items(section):
1742                     if key in places[section]: chgopt += 1
1743                     else: addopt += 1
1744                     if key in ("centroid", "location"):
1745                         places[section][key] = eval(value)
1746                     else:
1747                         places[section][key] = value
1748                 if addopt and chgopt:
1749                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1750                 elif addopt: logact += " (+%s options)" % addopt
1751                 elif chgopt: logact += " (!%s options)" % chgopt
1752         count += 1
1753         overrideslog.append("%s\n" % logact)
1754     overrideslog.sort()
1755     if os.path.exists(overrideslog_fn):
1756         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1757     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1758     import time
1759     overrideslog_fd.write(
1760         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
1761         '# use, copy, modify, and distribute this software is granted under terms\n'
1762         '# provided in the LICENSE file distributed with this software.\n\n'
1763         % time.gmtime().tm_year)
1764     overrideslog_fd.writelines(overrideslog)
1765     overrideslog_fd.close()
1766     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1767         count,
1768         added,
1769         removed,
1770         changed
1771     ) )
1772     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1773     print(
1774         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1775             estimate
1776     )
1777     count = 0
1778     milestones = list( range(51) )
1779     message = "   "
1780     sys.stdout.write(message)
1781     sys.stdout.flush()
1782     for fips in places:
1783         centroid = places[fips]["centroid"]
1784         if centroid:
1785             station = closest(centroid, stations, "location", 0.1)
1786         if station[0]:
1787             places[fips]["station"] = station
1788             count += 1
1789             if not count%100:
1790                 level = int(50*count/estimate)
1791                 if level in milestones:
1792                     for remaining in milestones[:milestones.index(level)+1]:
1793                         if remaining%5:
1794                             message = "."
1795                             sys.stdout.write(message)
1796                             sys.stdout.flush()
1797                         else:
1798                             message = "%s%%" % (remaining*2,)
1799                             sys.stdout.write(message)
1800                             sys.stdout.flush()
1801                         milestones.remove(remaining)
1802         if centroid:
1803             zone = closest(centroid, zones, "centroid", 0.1)
1804         if zone[0]:
1805             places[fips]["zone"] = zone
1806             count += 1
1807             if not count%100:
1808                 level = int(50*count/estimate)
1809                 if level in milestones:
1810                     for remaining in milestones[:milestones.index(level)+1]:
1811                         if remaining%5:
1812                             message = "."
1813                             sys.stdout.write(message)
1814                             sys.stdout.flush()
1815                         else:
1816                             message = "%s%%" % (remaining*2,)
1817                             sys.stdout.write(message)
1818                             sys.stdout.flush()
1819                         milestones.remove(remaining)
1820     for station in stations:
1821         if "location" in stations[station]:
1822             location = stations[station]["location"]
1823             if location:
1824                 zone = closest(location, zones, "centroid", 0.1)
1825             if zone[0]:
1826                 stations[station]["zone"] = zone
1827                 count += 1
1828                 if not count%100:
1829                     level = int(50*count/estimate)
1830                     if level in milestones:
1831                         for remaining in milestones[:milestones.index(level)+1]:
1832                             if remaining%5:
1833                                 message = "."
1834                                 sys.stdout.write(message)
1835                                 sys.stdout.flush()
1836                             else:
1837                                 message = "%s%%" % (remaining*2,)
1838                                 sys.stdout.write(message)
1839                                 sys.stdout.flush()
1840                             milestones.remove(remaining)
1841     for zcta in zctas.keys():
1842         centroid = zctas[zcta]["centroid"]
1843         if centroid:
1844             station = closest(centroid, stations, "location", 0.1)
1845         if station[0]:
1846             zctas[zcta]["station"] = station
1847             count += 1
1848             if not count%100:
1849                 level = int(50*count/estimate)
1850                 if level in milestones:
1851                     for remaining in milestones[ : milestones.index(level)+1 ]:
1852                         if remaining%5:
1853                             message = "."
1854                             sys.stdout.write(message)
1855                             sys.stdout.flush()
1856                         else:
1857                             message = "%s%%" % (remaining*2,)
1858                             sys.stdout.write(message)
1859                             sys.stdout.flush()
1860                         milestones.remove(remaining)
1861         if centroid:
1862             zone = closest(centroid, zones, "centroid", 0.1)
1863         if zone[0]:
1864             zctas[zcta]["zone"] = zone
1865             count += 1
1866             if not count%100:
1867                 level = int(50*count/estimate)
1868                 if level in milestones:
1869                     for remaining in milestones[:milestones.index(level)+1]:
1870                         if remaining%5:
1871                             message = "."
1872                             sys.stdout.write(message)
1873                             sys.stdout.flush()
1874                         else:
1875                             message = "%s%%" % (remaining*2,)
1876                             sys.stdout.write(message)
1877                             sys.stdout.flush()
1878                         milestones.remove(remaining)
1879     for zone in zones.keys():
1880         if "centroid" in zones[zone]:
1881             centroid = zones[zone]["centroid"]
1882             if centroid:
1883                 station = closest(centroid, stations, "location", 0.1)
1884             if station[0]:
1885                 zones[zone]["station"] = station
1886                 count += 1
1887                 if not count%100:
1888                     level = int(50*count/estimate)
1889                     if level in milestones:
1890                         for remaining in milestones[:milestones.index(level)+1]:
1891                             if remaining%5:
1892                                 message = "."
1893                                 sys.stdout.write(message)
1894                                 sys.stdout.flush()
1895                             else:
1896                                 message = "%s%%" % (remaining*2,)
1897                                 sys.stdout.write(message)
1898                                 sys.stdout.flush()
1899                             milestones.remove(remaining)
1900     for remaining in milestones:
1901         if remaining%5:
1902             message = "."
1903             sys.stdout.write(message)
1904             sys.stdout.flush()
1905         else:
1906             message = "%s%%" % (remaining*2,)
1907             sys.stdout.write(message)
1908             sys.stdout.flush()
1909     print("\n   done (%s correlations)." % count)
1910     message = "Writing %s..." % airports_fn
1911     sys.stdout.write(message)
1912     sys.stdout.flush()
1913     count = 0
1914     if os.path.exists(airports_fn):
1915         os.rename(airports_fn, "%s_old"%airports_fn)
1916     airports_fd = codecs.open(airports_fn, "w", "utf8")
1917     airports_fd.write(header)
1918     for airport in sorted( airports.keys() ):
1919         airports_fd.write("\n\n[%s]" % airport)
1920         for key, value in sorted( airports[airport].items() ):
1921             if type(value) is float: value = "%.7f"%value
1922             elif type(value) is tuple:
1923                 elements = []
1924                 for element in value:
1925                     if type(element) is float: elements.append("%.7f"%element)
1926                     else: elements.append( repr(element) )
1927                 value = "(%s)"%", ".join(elements)
1928             airports_fd.write( "\n%s = %s" % (key, value) )
1929         count += 1
1930     airports_fd.write("\n")
1931     airports_fd.close()
1932     print("done (%s sections)." % count)
1933     message = "Writing %s..." % places_fn
1934     sys.stdout.write(message)
1935     sys.stdout.flush()
1936     count = 0
1937     if os.path.exists(places_fn):
1938         os.rename(places_fn, "%s_old"%places_fn)
1939     places_fd = codecs.open(places_fn, "w", "utf8")
1940     places_fd.write(header)
1941     for fips in sorted( places.keys() ):
1942         places_fd.write("\n\n[%s]" % fips)
1943         for key, value in sorted( places[fips].items() ):
1944             if type(value) is float: value = "%.7f"%value
1945             elif type(value) is tuple:
1946                 elements = []
1947                 for element in value:
1948                     if type(element) is float: elements.append("%.7f"%element)
1949                     else: elements.append( repr(element) )
1950                 value = "(%s)"%", ".join(elements)
1951             places_fd.write( "\n%s = %s" % (key, value) )
1952         count += 1
1953     places_fd.write("\n")
1954     places_fd.close()
1955     print("done (%s sections)." % count)
1956     message = "Writing %s..." % stations_fn
1957     sys.stdout.write(message)
1958     sys.stdout.flush()
1959     count = 0
1960     if os.path.exists(stations_fn):
1961         os.rename(stations_fn, "%s_old"%stations_fn)
1962     stations_fd = codecs.open(stations_fn, "w", "utf-8")
1963     stations_fd.write(header)
1964     for station in sorted( stations.keys() ):
1965         stations_fd.write("\n\n[%s]" % station)
1966         for key, value in sorted( stations[station].items() ):
1967             if type(value) is float: value = "%.7f"%value
1968             elif type(value) is tuple:
1969                 elements = []
1970                 for element in value:
1971                     if type(element) is float: elements.append("%.7f"%element)
1972                     else: elements.append( repr(element) )
1973                 value = "(%s)"%", ".join(elements)
1974             if type(value) is bytes:
1975                 value = value.decode("utf-8")
1976             stations_fd.write( "\n%s = %s" % (key, value) )
1977         count += 1
1978     stations_fd.write("\n")
1979     stations_fd.close()
1980     print("done (%s sections)." % count)
1981     message = "Writing %s..." % zctas_fn
1982     sys.stdout.write(message)
1983     sys.stdout.flush()
1984     count = 0
1985     if os.path.exists(zctas_fn):
1986         os.rename(zctas_fn, "%s_old"%zctas_fn)
1987     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
1988     zctas_fd.write(header)
1989     for zcta in sorted( zctas.keys() ):
1990         zctas_fd.write("\n\n[%s]" % zcta)
1991         for key, value in sorted( zctas[zcta].items() ):
1992             if type(value) is float: value = "%.7f"%value
1993             elif type(value) is tuple:
1994                 elements = []
1995                 for element in value:
1996                     if type(element) is float: elements.append("%.7f"%element)
1997                     else: elements.append( repr(element) )
1998                 value = "(%s)"%", ".join(elements)
1999             zctas_fd.write( "\n%s = %s" % (key, value) )
2000         count += 1
2001     zctas_fd.write("\n")
2002     zctas_fd.close()
2003     print("done (%s sections)." % count)
2004     message = "Writing %s..." % zones_fn
2005     sys.stdout.write(message)
2006     sys.stdout.flush()
2007     count = 0
2008     if os.path.exists(zones_fn):
2009         os.rename(zones_fn, "%s_old"%zones_fn)
2010     zones_fd = codecs.open(zones_fn, "w", "utf8")
2011     zones_fd.write(header)
2012     for zone in sorted( zones.keys() ):
2013         zones_fd.write("\n\n[%s]" % zone)
2014         for key, value in sorted( zones[zone].items() ):
2015             if type(value) is float: value = "%.7f"%value
2016             elif type(value) is tuple:
2017                 elements = []
2018                 for element in value:
2019                     if type(element) is float: elements.append("%.7f"%element)
2020                     else: elements.append( repr(element) )
2021                 value = "(%s)"%", ".join(elements)
2022             zones_fd.write( "\n%s = %s" % (key, value) )
2023         count += 1
2024     zones_fd.write("\n")
2025     zones_fd.close()
2026     print("done (%s sections)." % count)
2027     message = "Starting QA check..."
2028     sys.stdout.write(message)
2029     sys.stdout.flush()
2030     airports = configparser.ConfigParser()
2031     if pyversion("3"):
2032         airports.read(airports_fn, encoding="utf-8")
2033     else:
2034         airports.read(airports_fn)
2035     places = configparser.ConfigParser()
2036     if pyversion("3"):
2037         places.read(places_fn, encoding="utf-8")
2038     else:
2039         places.read(places_fn)
2040     stations = configparser.ConfigParser()
2041     if pyversion("3"):
2042         stations.read(stations_fn, encoding="utf-8")
2043     else:
2044         stations.read(stations_fn)
2045     zctas = configparser.ConfigParser()
2046     if pyversion("3"):
2047         zctas.read(zctas_fn, encoding="utf-8")
2048     else:
2049         zctas.read(zctas_fn)
2050     zones = configparser.ConfigParser()
2051     if pyversion("3"):
2052         zones.read(zones_fn, encoding="utf-8")
2053     else:
2054         zones.read(zones_fn)
2055     qalog = []
2056     places_nocentroid = 0
2057     places_nodescription = 0
2058     for place in sorted( places.sections() ):
2059         if not places.has_option(place, "centroid"):
2060             qalog.append("%s: no centroid\n" % place)
2061             places_nocentroid += 1
2062         if not places.has_option(place, "description"):
2063             qalog.append("%s: no description\n" % place)
2064             places_nodescription += 1
2065     stations_nodescription = 0
2066     stations_nolocation = 0
2067     stations_nometar = 0
2068     for station in sorted( stations.sections() ):
2069         if not stations.has_option(station, "description"):
2070             qalog.append("%s: no description\n" % station)
2071             stations_nodescription += 1
2072         if not stations.has_option(station, "location"):
2073             qalog.append("%s: no location\n" % station)
2074             stations_nolocation += 1
2075         if not stations.has_option(station, "metar"):
2076             qalog.append("%s: no metar\n" % station)
2077             stations_nometar += 1
2078     airports_badstation = 0
2079     airports_nostation = 0
2080     for airport in sorted( airports.sections() ):
2081         if not airports.has_option(airport, "station"):
2082             qalog.append("%s: no station\n" % airport)
2083             airports_nostation += 1
2084         else:
2085             station = airports.get(airport, "station")
2086             if station not in stations.sections():
2087                 qalog.append( "%s: bad station %s\n" % (airport, station) )
2088                 airports_badstation += 1
2089     zctas_nocentroid = 0
2090     for zcta in sorted( zctas.sections() ):
2091         if not zctas.has_option(zcta, "centroid"):
2092             qalog.append("%s: no centroid\n" % zcta)
2093             zctas_nocentroid += 1
2094     zones_nocentroid = 0
2095     zones_nodescription = 0
2096     zones_noforecast = 0
2097     zones_overlapping = 0
2098     zonetable = {}
2099     for zone in zones.sections():
2100         if zones.has_option(zone, "centroid"):
2101             zonetable[zone] = {
2102                 "centroid": eval( zones.get(zone, "centroid") )
2103             }
2104     for zone in sorted( zones.sections() ):
2105         if zones.has_option(zone, "centroid"):
2106             zonetable_local = zonetable.copy()
2107             del( zonetable_local[zone] )
2108             centroid = eval( zones.get(zone, "centroid") )
2109             if centroid:
2110                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2111             if nearest[1]*radian_to_km < 1:
2112                 qalog.append( "%s: within one km of %s\n" % (
2113                     zone,
2114                     nearest[0]
2115                 ) )
2116                 zones_overlapping += 1
2117         else:
2118             qalog.append("%s: no centroid\n" % zone)
2119             zones_nocentroid += 1
2120         if not zones.has_option(zone, "description"):
2121             qalog.append("%s: no description\n" % zone)
2122             zones_nodescription += 1
2123         if not zones.has_option(zone, "zone_forecast"):
2124             qalog.append("%s: no forecast\n" % zone)
2125             zones_noforecast += 1
2126     if os.path.exists(qalog_fn):
2127         os.rename(qalog_fn, "%s_old"%qalog_fn)
2128     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2129     import time
2130     qalog_fd.write(
2131         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
2132         '# use, copy, modify, and distribute this software is granted under terms\n'
2133         '# provided in the LICENSE file distributed with this software.\n\n'
2134         % time.gmtime().tm_year)
2135     qalog_fd.writelines(qalog)
2136     qalog_fd.close()
2137     if qalog:
2138         print("issues found (see %s for details):"%qalog_fn)
2139         if airports_badstation:
2140             print("   %s airports with invalid station"%airports_badstation)
2141         if airports_nostation:
2142             print("   %s airports with no station"%airports_nostation)
2143         if places_nocentroid:
2144             print("   %s places with no centroid"%places_nocentroid)
2145         if places_nodescription:
2146             print("   %s places with no description"%places_nodescription)
2147         if stations_nodescription:
2148             print("   %s stations with no description"%stations_nodescription)
2149         if stations_nolocation:
2150             print("   %s stations with no location"%stations_nolocation)
2151         if stations_nometar:
2152             print("   %s stations with no METAR"%stations_nometar)
2153         if zctas_nocentroid:
2154             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2155         if zones_nocentroid:
2156             print("   %s zones with no centroid"%zones_nocentroid)
2157         if zones_nodescription:
2158             print("   %s zones with no description"%zones_nodescription)
2159         if zones_noforecast:
2160             print("   %s zones with no forecast"%zones_noforecast)
2161         if zones_overlapping:
2162             print("   %s zones within one km of another"%zones_overlapping)
2163     else: print("no issues found.")
2164     print("Indexing complete!")