weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2024 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.4.4"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 class Selections:
  15     """An object to contain selection data."""
  16     def __init__(self):
  17         """Store the config, options and arguments."""
  18         self.config = get_config()
  19         self.options, self.arguments = get_options(self.config)
  20         if self.get_bool("cache") and self.get_bool("cache_search") \
  21             and not self.get_bool("longlist"):
  22             integrate_search_cache(
  23                 self.config,
  24                 self.get("cachedir"),
  25                 self.get("setpath")
  26             )
  27         if not self.arguments:
  28             if "id" in self.options.__dict__ \
  29                 and self.options.__dict__["id"]:
  30                 self.arguments.append( self.options.__dict__["id"] )
  31                 del( self.options.__dict__["id"] )
  32                 import sys
  33                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  34                 sys.stderr.write(message)
  35             elif "city" in self.options.__dict__ \
  36                 and self.options.__dict__["city"] \
  37                 and "st" in self.options.__dict__ \
  38                 and self.options.__dict__["st"]:
  39                 self.arguments.append(
  40                     "^%s city, %s" % (
  41                         self.options.__dict__["city"],
  42                         self.options.__dict__["st"]
  43                     )
  44                 )
  45                 del( self.options.__dict__["city"] )
  46                 del( self.options.__dict__["st"] )
  47                 import sys
  48                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  49                 sys.stderr.write(message)
  50     def get(self, option, argument=None):
  51         """Retrieve data from the config or options."""
  52         if argument:
  53             if self.config.has_section(argument) and (
  54                 self.config.has_option(argument, "city") \
  55                     or self.config.has_option(argument, "id") \
  56                     or self.config.has_option(argument, "st")
  57             ):
  58                 self.config.remove_section(argument)
  59                 import sys
  60                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  61                 sys.stderr.write(message)
  62             if not self.config.has_section(argument):
  63                 guessed = guess(
  64                     argument,
  65                     path=self.get("setpath"),
  66                     info=self.get("info"),
  67                     cache_search=(
  68                         self.get("cache") and self.get("cache_search")
  69                     ),
  70                     cachedir=self.get("cachedir"),
  71                     quiet=self.get_bool("quiet")
  72                 )
  73                 self.config.add_section(argument)
  74                 for item in guessed.items():
  75                     self.config.set(argument, *item)
  76             if self.config.has_option(argument, option):
  77                 return self.config.get(argument, option)
  78         if option in self.options.__dict__:
  79             return self.options.__dict__[option]
  80         import sys
  81         message = "WARNING: no URI defined for %s\n" % option
  82         sys.stderr.write(message)
  83         return None
  84     def get_bool(self, option, argument=None):
  85         """Get data and coerce to a boolean if necessary."""
  86         # Mimic configparser's getboolean() method by treating
  87         # false/no/off/0 as False and true/yes/on/1 as True values,
  88         # case-insensitively
  89         value = self.get(option, argument)
  90         if isinstance(value, bool):
  91             return value
  92         if isinstance(value, str):
  93             vlower = value.lower()
  94             if vlower in ('false', 'no', 'off', '0'):
  95                 return False
  96             elif vlower in ('true', 'yes', 'on', '1'):
  97                 return True
  98         raise ValueError("Not a boolean: %s" % value)
  99     def getint(self, option, argument=None):
 100         """Get data and coerce to an integer if necessary."""
 101         value = self.get(option, argument)
 102         if value: return int(value)
 103         else: return 0
 104
 105 def average(coords):
 106     """Average a list of coordinates."""
 107     x = 0
 108     y = 0
 109     for coord in coords:
 110         x += coord[0]
 111         y += coord[1]
 112     count = len(coords)
 113     return (x/count, y/count)
 114
 115 def filter_units(line, units="imperial"):
 116     """Filter or convert units in a line of text between US/UK and metric."""
 117     import re
 118     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 119     # "X in. Hg (Y hPa)"
 120     dual_p = re.match(
 121         r"(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 122         line
 123     )
 124     if dual_p:
 125         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 126         if units == "imperial": line = preamble + in_hg + trailer
 127         elif units == "metric": line = preamble + hpa + trailer
 128     # filter lines with both temperatures in the form of "X F (Y C)"
 129     dual_t = re.match(
 130         r"(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 131         line
 132     )
 133     if dual_t:
 134         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 135         if units == "imperial": line = preamble + fahrenheit + trailer
 136         elif units == "metric": line = preamble + celsius + trailer
 137     # if metric is desired, convert distances in the form of "X mile(s)" to
 138     # "Y kilometer(s)"
 139     if units == "metric":
 140         imperial_d = re.match(
 141             r"(.* )(\d+)( mile\(s\))(.*)",
 142             line
 143         )
 144         if imperial_d:
 145             preamble, mi, m_u, trailer = imperial_d.groups()
 146             line = preamble + str(int(round(int(mi)*1.609344))) \
 147                 + " kilometer(s)" + trailer
 148     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 149     # desired, convert to "Z KPH"
 150     imperial_s = re.match(
 151         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 152         line
 153     )
 154     if imperial_s:
 155         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 156         if units == "imperial": line = preamble + mph + m_u + trailer
 157         elif units == "metric":
 158             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 159                 trailer
 160     imperial_s = re.match(
 161         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 162         line
 163     )
 164     if imperial_s:
 165         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 166         if units == "imperial": line = preamble + mph + m_u + trailer
 167         elif units == "metric":
 168             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 169                 trailer
 170     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 171     # metric is desired, convert to "Y C"
 172     imperial_t = re.match(
 173         r"(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 174         line
 175     )
 176     if imperial_t:
 177         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 178         if units == "imperial":
 179             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 180         elif units == "metric":
 181             line = preamble + parameter \
 182                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 183                 + trailer
 184     # hand off the resulting line
 185     return line
 186
 187 def get_uri(
 188     uri,
 189     ignore_fail=False,
 190     cache_data=False,
 191     cacheage=900,
 192     cachedir="."
 193 ):
 194     """Return a string containing the results of a URI GET."""
 195     import os, time, urllib, urllib.error, urllib.request
 196     if cache_data:
 197         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 198         if not os.path.exists(dcachedir):
 199             try: os.makedirs(dcachedir)
 200             except (IOError, OSError): pass
 201         dcache_fn = os.path.join(
 202             dcachedir,
 203             uri.split(":",1)[1].replace("/","_")
 204         )
 205     now = time.time()
 206     if cache_data and os.access(dcache_fn, os.R_OK) \
 207         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 208         dcache_fd = open(dcache_fn)
 209         data = dcache_fd.read()
 210         dcache_fd.close()
 211     else:
 212         try:
 213             data = urllib.request.urlopen(uri).read().decode("utf-8")
 214         except urllib.error.URLError:
 215             if ignore_fail: return ""
 216             import os, sys
 217             sys.stderr.write("%s error: failed to retrieve\n   %s\n\n" % (
 218                 os.path.basename( sys.argv[0] ), uri))
 219             raise
 220         # Some data sources are HTML with the plain text wrapped in pre tags
 221         if "<pre>" in data:
 222             data = data[data.find("<pre>")+5:data.find("</pre>")]
 223         if cache_data:
 224             try:
 225                 import codecs
 226                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 227                 dcache_fd.write(data)
 228                 dcache_fd.close()
 229             except (IOError, OSError): pass
 230     return data
 231
 232 def get_metar(
 233     uri=None,
 234     verbose=False,
 235     quiet=False,
 236     headers=None,
 237     imperial=False,
 238     metric=False,
 239     cache_data=False,
 240     cacheage=900,
 241     cachedir="."
 242 ):
 243     """Return a summarized METAR for the specified station."""
 244     if not uri:
 245         import os, sys
 246         message = "%s error: METAR URI required for conditions\n" % \
 247             os.path.basename( sys.argv[0] )
 248         sys.stderr.write(message)
 249         sys.exit(1)
 250     metar = get_uri(
 251         uri,
 252         cache_data=cache_data,
 253         cacheage=cacheage,
 254         cachedir=cachedir
 255     )
 256     if type(metar) is bytes: metar = metar.decode("utf-8")
 257     if verbose: return metar
 258     else:
 259         import re
 260         lines = metar.split("\n")
 261         if not headers:
 262             headers = \
 263                 "relative_humidity," \
 264                 + "precipitation_last_hour," \
 265                 + "sky conditions," \
 266                 + "temperature," \
 267                 + "heat index," \
 268                 + "windchill," \
 269                 + "weather," \
 270                 + "wind"
 271         headerlist = headers.lower().replace("_"," ").split(",")
 272         output = []
 273         if not quiet:
 274             title = "Current conditions at %s"
 275             place = lines[0].split(", ")
 276             if len(place) > 1:
 277                 place = "%s, %s" % ( place[0].title(), place[1] )
 278             else: place = "<UNKNOWN>"
 279             output.append(title%place)
 280             output.append("Last updated " + lines[1])
 281         header_match = False
 282         for header in headerlist:
 283             for line in lines:
 284                 if line.lower().startswith(header + ":"):
 285                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 286                     if imperial: line = filter_units(line, units="imperial")
 287                     elif metric: line = filter_units(line, units="metric")
 288                     if quiet: output.append(line)
 289                     else: output.append("   " + line)
 290                     header_match = True
 291         if not header_match:
 292             output.append(
 293                 "(no conditions matched your header list, try with --verbose)"
 294             )
 295         return "\n".join(output)
 296
 297 def get_alert(
 298     uri=None,
 299     verbose=False,
 300     quiet=False,
 301     cache_data=False,
 302     cacheage=900,
 303     cachedir=".",
 304     delay=1
 305 ):
 306     """Return alert notice for the specified URI."""
 307     if not uri:
 308         return ""
 309     alert = get_uri(
 310         uri,
 311         ignore_fail=True,
 312         cache_data=cache_data,
 313         cacheage=cacheage,
 314         cachedir=cachedir
 315     ).strip()
 316     if type(alert) is bytes: alert = alert.decode("utf-8")
 317     if alert:
 318         if verbose: return alert
 319         else:
 320             import re
 321             if re.search(r"\nNational Weather Service", alert):
 322                 muted = True
 323             else:
 324                 muted = False
 325             expirycheck = re.search(r"Expires:([0-9]{12})", alert)
 326             if expirycheck:
 327                 # only report alerts and forecasts that expired less than delay
 328                 # hours ago
 329                 import datetime, zoneinfo
 330                 expiration = datetime.datetime.fromisoformat(
 331                     "%s-%s-%sT%s:%s" % (
 332                         expirycheck[1][:4],
 333                         expirycheck[1][4:6],
 334                         expirycheck[1][6:8],
 335                         expirycheck[1][8:10],
 336                         expirycheck[1][-2:],
 337                     )).replace(tzinfo=zoneinfo.ZoneInfo("UTC"))
 338                 now = datetime.datetime.now(tz=zoneinfo.ZoneInfo("UTC"))
 339                 if now - expiration > datetime.timedelta(hours=delay):
 340                     return ""
 341             lines = alert.split("\n")
 342             output = []
 343             for line in lines:
 344                 if muted and line.startswith("National Weather Service"):
 345                     muted = False
 346                     line = ""
 347                 elif line == "&&":
 348                     line = ""
 349                 elif line == "$$":
 350                     muted = True
 351                 if line and not muted:
 352                     if quiet: output.append(line)
 353                     else: output.append("   " + line)
 354             return "\n".join(output)
 355
 356 def get_options(config):
 357     """Parse the options passed on the command line."""
 358
 359     # for optparse's builtin -h/--help option
 360     usage = \
 361         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 362
 363     # for optparse's builtin --version option
 364     verstring = "%prog " + weather_version
 365
 366     # create the parser
 367     import optparse
 368     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 369     # separate options object from list of arguments and return both
 370
 371     # the -a/--alert option
 372     if config.has_option("default", "alert"):
 373         default_alert = config.getboolean("default", "alert")
 374     else: default_alert = False
 375     option_parser.add_option("-a", "--alert",
 376         dest="alert",
 377         action="store_true",
 378         default=default_alert,
 379         help="include local alert notices")
 380
 381     # the --atypes option
 382     if config.has_option("default", "atypes"):
 383         default_atypes = config.get("default", "atypes")
 384     else:
 385         default_atypes = \
 386             "coastal_flood_statement," \
 387             + "flash_flood_statement," \
 388             + "flash_flood_warning," \
 389             + "flash_flood_watch," \
 390             + "flood_warning," \
 391             + "severe_thunderstorm_warning," \
 392             + "severe_weather_statement," \
 393             + "special_weather_statement," \
 394             + "tornado," \
 395             + "urgent_weather_message"
 396     option_parser.add_option("--atypes",
 397         dest="atypes",
 398         default=default_atypes,
 399         help="list of alert notification types to display")
 400
 401     # the --build-sets option
 402     option_parser.add_option("--build-sets",
 403         dest="build_sets",
 404         action="store_true",
 405         default=False,
 406         help="(re)build location correlation sets")
 407
 408     # the --cacheage option
 409     if config.has_option("default", "cacheage"):
 410         default_cacheage = config.getint("default", "cacheage")
 411     else: default_cacheage = 900
 412     option_parser.add_option("--cacheage",
 413         dest="cacheage",
 414         default=default_cacheage,
 415         help="duration in seconds to refresh cached data")
 416
 417     # the --cachedir option
 418     if config.has_option("default", "cachedir"):
 419         default_cachedir = config.get("default", "cachedir")
 420     else: default_cachedir = "~/.weather"
 421     option_parser.add_option("--cachedir",
 422         dest="cachedir",
 423         default=default_cachedir,
 424         help="directory for storing cached searches and data")
 425
 426     # the --delay option
 427     if config.has_option("default", "delay"):
 428         default_delay = config.getint("default", "delay")
 429     else: default_delay = 1
 430     option_parser.add_option("--delay",
 431         dest="delay",
 432         default=default_delay,
 433         help="hours to delay alert and forecast expiration")
 434
 435     # the -f/--forecast option
 436     if config.has_option("default", "forecast"):
 437         default_forecast = config.getboolean("default", "forecast")
 438     else: default_forecast = False
 439     option_parser.add_option("-f", "--forecast",
 440         dest="forecast",
 441         action="store_true",
 442         default=default_forecast,
 443         help="include a local forecast")
 444
 445     # the --headers option
 446     if config.has_option("default", "headers"):
 447         default_headers = config.get("default", "headers")
 448     else:
 449         default_headers = \
 450             "temperature," \
 451             + "relative_humidity," \
 452             + "wind," \
 453             + "heat_index," \
 454             + "windchill," \
 455             + "weather," \
 456             + "sky_conditions," \
 457             + "precipitation_last_hour"
 458     option_parser.add_option("--headers",
 459         dest="headers",
 460         default=default_headers,
 461         help="list of conditions headers to display")
 462
 463     # the --imperial option
 464     if config.has_option("default", "imperial"):
 465         default_imperial = config.getboolean("default", "imperial")
 466     else: default_imperial = False
 467     option_parser.add_option("--imperial",
 468         dest="imperial",
 469         action="store_true",
 470         default=default_imperial,
 471         help="filter/convert conditions for US/UK units")
 472
 473     # the --info option
 474     option_parser.add_option("--info",
 475         dest="info",
 476         action="store_true",
 477         default=False,
 478         help="output detailed information for your search")
 479
 480     # the -l/--list option
 481     option_parser.add_option("-l", "--list",
 482         dest="list",
 483         action="store_true",
 484         default=False,
 485         help="list all configured aliases and cached searches")
 486
 487     # the --longlist option
 488     option_parser.add_option("--longlist",
 489         dest="longlist",
 490         action="store_true",
 491         default=False,
 492         help="display details of all configured aliases")
 493
 494     # the -m/--metric option
 495     if config.has_option("default", "metric"):
 496         default_metric = config.getboolean("default", "metric")
 497     else: default_metric = False
 498     option_parser.add_option("-m", "--metric",
 499         dest="metric",
 500         action="store_true",
 501         default=default_metric,
 502         help="filter/convert conditions for metric units")
 503
 504     # the -n/--no-conditions option
 505     if config.has_option("default", "conditions"):
 506         default_conditions = config.getboolean("default", "conditions")
 507     else: default_conditions = True
 508     option_parser.add_option("-n", "--no-conditions",
 509         dest="conditions",
 510         action="store_false",
 511         default=default_conditions,
 512         help="disable output of current conditions")
 513
 514     # the --no-cache option
 515     if config.has_option("default", "cache"):
 516         default_cache = config.getboolean("default", "cache")
 517     else: default_cache = True
 518     option_parser.add_option("--no-cache",
 519         dest="cache",
 520         action="store_false",
 521         default=True,
 522         help="disable all caching (searches and data)")
 523
 524     # the --no-cache-data option
 525     if config.has_option("default", "cache_data"):
 526         default_cache_data = config.getboolean("default", "cache_data")
 527     else: default_cache_data = True
 528     option_parser.add_option("--no-cache-data",
 529         dest="cache_data",
 530         action="store_false",
 531         default=True,
 532         help="disable retrieved data caching")
 533
 534     # the --no-cache-search option
 535     if config.has_option("default", "cache_search"):
 536         default_cache_search = config.getboolean("default", "cache_search")
 537     else: default_cache_search = True
 538     option_parser.add_option("--no-cache-search",
 539         dest="cache_search",
 540         action="store_false",
 541         default=True,
 542         help="disable search result caching")
 543
 544     # the -q/--quiet option
 545     if config.has_option("default", "quiet"):
 546         default_quiet = config.getboolean("default", "quiet")
 547     else: default_quiet = False
 548     option_parser.add_option("-q", "--quiet",
 549         dest="quiet",
 550         action="store_true",
 551         default=default_quiet,
 552         help="skip preambles and don't indent")
 553
 554     # the --setpath option
 555     if config.has_option("default", "setpath"):
 556         default_setpath = config.get("default", "setpath")
 557     else: default_setpath = ".:~/.weather"
 558     option_parser.add_option("--setpath",
 559         dest="setpath",
 560         default=default_setpath,
 561         help="directory search path for correlation sets")
 562
 563     # the -v/--verbose option
 564     if config.has_option("default", "verbose"):
 565         default_verbose = config.getboolean("default", "verbose")
 566     else: default_verbose = False
 567     option_parser.add_option("-v", "--verbose",
 568         dest="verbose",
 569         action="store_true",
 570         default=default_verbose,
 571         help="show full decoded feeds")
 572
 573     # deprecated options
 574     if config.has_option("default", "city"):
 575         default_city = config.get("default", "city")
 576     else: default_city = ""
 577     option_parser.add_option("-c", "--city",
 578         dest="city",
 579         default=default_city,
 580         help=optparse.SUPPRESS_HELP)
 581     if config.has_option("default", "id"):
 582         default_id = config.get("default", "id")
 583     else: default_id = ""
 584     option_parser.add_option("-i", "--id",
 585         dest="id",
 586         default=default_id,
 587         help=optparse.SUPPRESS_HELP)
 588     if config.has_option("default", "st"):
 589         default_st = config.get("default", "st")
 590     else: default_st = ""
 591     option_parser.add_option("-s", "--st",
 592         dest="st",
 593         default=default_st,
 594         help=optparse.SUPPRESS_HELP)
 595
 596     options, arguments = option_parser.parse_args()
 597     return options, arguments
 598
 599 def get_config():
 600     """Parse the aliases and configuration."""
 601     import configparser, os
 602     config = configparser.ConfigParser()
 603     rcfiles = [
 604         "/etc/weatherrc",
 605         "/etc/weather/weatherrc",
 606         os.path.expanduser("~/.weather/weatherrc"),
 607         os.path.expanduser("~/.weatherrc"),
 608         "weatherrc"
 609         ]
 610     for rcfile in rcfiles:
 611         if os.access(rcfile, os.R_OK):
 612             config.read(rcfile, encoding="utf-8")
 613     for section in config.sections():
 614         if section != section.lower():
 615             if config.has_section(section.lower()):
 616                 config.remove_section(section.lower())
 617             config.add_section(section.lower())
 618             for option,value in config.items(section):
 619                 config.set(section.lower(), option, value)
 620     return config
 621
 622 def integrate_search_cache(config, cachedir, setpath):
 623     """Add cached search results into the configuration."""
 624     import configparser, os, time
 625     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 626     if not os.access(scache_fn, os.R_OK): return config
 627     scache_fd = open(scache_fn)
 628     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 629     scache_fd.close()
 630     now = time.time()
 631     datafiles = data_index(setpath)
 632     if datafiles:
 633         data_freshness = sorted(
 634             [ x[1] for x in datafiles.values() ],
 635             reverse=True
 636         )[0]
 637     else: data_freshness = now
 638     if created < data_freshness <= now:
 639         try:
 640             os.remove(scache_fn)
 641             print( "[clearing outdated %s]" % scache_fn )
 642         except (IOError, OSError):
 643             pass
 644         return config
 645     scache = configparser.ConfigParser()
 646     scache.read(scache_fn, encoding="utf-8")
 647     for section in scache.sections():
 648         if not config.has_section(section):
 649             config.add_section(section)
 650             for option,value in scache.items(section):
 651                 config.set(section, option, value)
 652     return config
 653
 654 def list_aliases(config, detail=False):
 655     """Return a formatted list of aliases defined in the config."""
 656     if detail:
 657         output = "\n# configured alias details..."
 658         for section in sorted(config.sections()):
 659             output += "\n\n[%s]" % section
 660             for item in sorted(config.items(section)):
 661                 output += "\n%s = %s" % item
 662         output += "\n"
 663     else:
 664         output = "configured aliases and cached searches..."
 665         for section in sorted(config.sections()):
 666             if config.has_option(section, "description"):
 667                 description = config.get(section, "description")
 668             else: description = "(no description provided)"
 669             output += "\n   %s: %s" % (section, description)
 670     return output
 671
 672 def data_index(path):
 673     import os
 674     datafiles = {}
 675     for filename in ("airports", "places", "stations", "zctas", "zones"):
 676         for dirname in path.split(":"):
 677             for extension in ("", ".gz", ".txt"):
 678                 candidate = os.path.expanduser(
 679                     os.path.join( dirname, "".join( (filename, extension) ) )
 680                 )
 681                 if os.path.exists(candidate):
 682                     datafiles[filename] = (
 683                         candidate,
 684                         os.stat(candidate).st_mtime
 685                     )
 686                     break
 687             if filename in datafiles:
 688                 break
 689     return datafiles
 690
 691 def guess(
 692     expression,
 693     path=".",
 694     max_results=20,
 695     info=False,
 696     cache_search=False,
 697     cacheage=900,
 698     cachedir=".",
 699     quiet=False
 700 ):
 701     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 702     import codecs, configparser, datetime, time, os, re, sys
 703     datafiles = data_index(path)
 704     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 705     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 706     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 707     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 708     elif re.match(
 709         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 710         expression
 711     ):
 712         searchtype = "coordinates"
 713     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 714     else:
 715         searchtype = "name"
 716         cache_search = False
 717     if cache_search: action = "caching"
 718     else: action = "using"
 719     if info:
 720         scores = [
 721             (0.005, "bad"),
 722             (0.025, "poor"),
 723             (0.160, "suspect"),
 724             (0.500, "mediocre"),
 725             (0.840, "good"),
 726             (0.975, "great"),
 727             (0.995, "excellent"),
 728             (1.000, "ideal"),
 729         ]
 730     if not quiet: print("Searching via %s..."%searchtype)
 731     stations = configparser.ConfigParser()
 732     dataname = "stations"
 733     if dataname in datafiles:
 734         datafile = datafiles[dataname][0]
 735         if datafile.endswith(".gz"):
 736             import gzip
 737             stations.read_string( gzip.open(datafile).read().decode("utf-8") )
 738         else:
 739             stations.read(datafile, encoding="utf-8")
 740     else:
 741         message = "%s error: can't find \"%s\" data file\n" % (
 742             os.path.basename( sys.argv[0] ),
 743             dataname
 744         )
 745         sys.stderr.write(message)
 746         exit(1)
 747     zones = configparser.ConfigParser()
 748     dataname = "zones"
 749     if dataname in datafiles:
 750         datafile = datafiles[dataname][0]
 751         if datafile.endswith(".gz"):
 752             import gzip
 753             zones.read_string( gzip.open(datafile).read().decode("utf-8") )
 754         else:
 755             zones.read(datafile, encoding="utf-8")
 756     else:
 757         message = "%s error: can't find \"%s\" data file\n" % (
 758             os.path.basename( sys.argv[0] ),
 759             dataname
 760         )
 761         sys.stderr.write(message)
 762         exit(1)
 763     search = None
 764     station = ("", 0)
 765     zone = ("", 0)
 766     dataset = None
 767     possibilities = []
 768     uris = {}
 769     if searchtype == "airport":
 770         expression = expression.lower()
 771         airports = configparser.ConfigParser()
 772         dataname = "airports"
 773         if dataname in datafiles:
 774             datafile = datafiles[dataname][0]
 775             if datafile.endswith(".gz"):
 776                 import gzip
 777                 airports.read_string(
 778                     gzip.open(datafile).read().decode("utf-8") )
 779             else:
 780                 airports.read(datafile, encoding="utf-8")
 781         else:
 782             message = "%s error: can't find \"%s\" data file\n" % (
 783                 os.path.basename( sys.argv[0] ),
 784                 dataname
 785             )
 786             sys.stderr.write(message)
 787             exit(1)
 788         if airports.has_section(expression) \
 789             and airports.has_option(expression, "station"):
 790             search = (expression, "IATA/FAA airport code %s" % expression)
 791             station = ( airports.get(expression, "station"), 0 )
 792             if stations.has_option(station[0], "zone"):
 793                 zone = eval( stations.get(station[0], "zone") )
 794                 dataset = stations
 795             if not ( info or quiet ) \
 796                 and stations.has_option( station[0], "description" ):
 797                 print(
 798                     "[%s result %s]" % (
 799                         action,
 800                         stations.get(station[0], "description")
 801                     )
 802                 )
 803         else:
 804             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 805                 expression,
 806                 datafiles["airports"][0]
 807             )
 808             sys.stderr.write(message)
 809             exit(1)
 810     elif searchtype == "station":
 811         expression = expression.lower()
 812         if stations.has_section(expression):
 813             station = (expression, 0)
 814             if not search:
 815                 search = (expression, "ICAO station code %s" % expression)
 816             if stations.has_option(expression, "zone"):
 817                 zone = eval( stations.get(expression, "zone") )
 818                 dataset = stations
 819             if not ( info or quiet ) \
 820                 and stations.has_option(expression, "description"):
 821                 print(
 822                     "[%s result %s]" % (
 823                         action,
 824                         stations.get(expression, "description")
 825                     )
 826                 )
 827         else:
 828             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 829                 expression,
 830                 datafiles["stations"][0]
 831             )
 832             sys.stderr.write(message)
 833             exit(1)
 834     elif searchtype == "zone":
 835         expression = expression.lower()
 836         if zones.has_section(expression) \
 837             and zones.has_option(expression, "station"):
 838             zone = (expression, 0)
 839             station = eval( zones.get(expression, "station") )
 840             dataset = zones
 841             search = (expression, "NWS/NOAA weather zone %s" % expression)
 842             if not ( info or quiet ) \
 843                 and zones.has_option(expression, "description"):
 844                 print(
 845                     "[%s result %s]" % (
 846                         action,
 847                         zones.get(expression, "description")
 848                     )
 849                 )
 850         else:
 851             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 852                 expression,
 853                 datafiles["zones"][0]
 854             )
 855             sys.stderr.write(message)
 856             exit(1)
 857     elif searchtype == "ZCTA":
 858         zctas = configparser.ConfigParser()
 859         dataname = "zctas"
 860         if dataname in datafiles:
 861             datafile = datafiles[dataname][0]
 862             if datafile.endswith(".gz"):
 863                 import gzip
 864                 zctas.read_string( gzip.open(datafile).read().decode("utf-8") )
 865             else:
 866                 zctas.read(datafile, encoding="utf-8")
 867         else:
 868             message = "%s error: can't find \"%s\" data file\n" % (
 869                 os.path.basename( sys.argv[0] ),
 870                 dataname
 871             )
 872             sys.stderr.write(message)
 873             exit(1)
 874         dataset = zctas
 875         if zctas.has_section(expression) \
 876             and zctas.has_option(expression, "station"):
 877             station = eval( zctas.get(expression, "station") )
 878             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 879             if zctas.has_option(expression, "zone"):
 880                 zone = eval( zctas.get(expression, "zone") )
 881         else:
 882             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 883                 expression,
 884                 datafiles["zctas"][0]
 885             )
 886             sys.stderr.write(message)
 887             exit(1)
 888     elif searchtype == "coordinates":
 889         search = (expression, "Geographic coordinates %s" % expression)
 890         stationtable = {}
 891         for station in stations.sections():
 892             if stations.has_option(station, "location"):
 893                 stationtable[station] = {
 894                     "location": eval( stations.get(station, "location") )
 895                 }
 896         station = closest( gecos(expression), stationtable, "location", 0.1 )
 897         if not station[0]:
 898             message = "No ICAO weather station found near %s.\n" % expression
 899             sys.stderr.write(message)
 900             exit(1)
 901         zonetable = {}
 902         for zone in zones.sections():
 903             if zones.has_option(zone, "centroid"):
 904                 zonetable[zone] = {
 905                     "centroid": eval( zones.get(zone, "centroid") )
 906                 }
 907         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 908         if not zone[0]:
 909             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 910                 % expression
 911             sys.stderr.write(message)
 912     elif searchtype in ("FIPS", "name"):
 913         places = configparser.ConfigParser()
 914         dataname = "places"
 915         if dataname in datafiles:
 916             datafile = datafiles[dataname][0]
 917             if datafile.endswith(".gz"):
 918                 import gzip
 919                 places.read_string( gzip.open(datafile).read().decode("utf-8") )
 920             else:
 921                 places.read(datafile, encoding="utf-8")
 922         else:
 923             message = "%s error: can't find \"%s\" data file\n" % (
 924                 os.path.basename( sys.argv[0] ),
 925                 dataname
 926             )
 927             sys.stderr.write(message)
 928             exit(1)
 929         dataset = places
 930         place = expression.lower()
 931         if places.has_section(place) and places.has_option(place, "station"):
 932             station = eval( places.get(place, "station") )
 933             search = (expression, "Census Place %s" % expression)
 934             if places.has_option(place, "description"):
 935                 search = (
 936                     search[0],
 937                     search[1] + ", %s" % places.get(place, "description")
 938                 )
 939             if places.has_option(place, "zone"):
 940                 zone = eval( places.get(place, "zone") )
 941             if not ( info or quiet ) \
 942                 and places.has_option(place, "description"):
 943                 print(
 944                     "[%s result %s]" % (
 945                         action,
 946                         places.get(place, "description")
 947                     )
 948                 )
 949         else:
 950             for place in places.sections():
 951                 if places.has_option(place, "description") \
 952                     and places.has_option(place, "station") \
 953                     and re.search(
 954                         expression,
 955                         places.get(place, "description"),
 956                         re.I
 957                     ):
 958                         possibilities.append(place)
 959             for place in stations.sections():
 960                 if stations.has_option(place, "description") \
 961                     and re.search(
 962                         expression,
 963                         stations.get(place, "description"),
 964                         re.I
 965                     ):
 966                         possibilities.append(place)
 967             for place in zones.sections():
 968                 if zones.has_option(place, "description") \
 969                     and zones.has_option(place, "station") \
 970                     and re.search(
 971                         expression,
 972                         zones.get(place, "description"),
 973                         re.I
 974                     ):
 975                         possibilities.append(place)
 976             if len(possibilities) == 1:
 977                 place = possibilities[0]
 978                 if places.has_section(place):
 979                     station = eval( places.get(place, "station") )
 980                     description = places.get(place, "description")
 981                     if places.has_option(place, "zone"):
 982                         zone = eval( places.get(place, "zone" ) )
 983                     search = ( expression, "%s: %s" % (place, description) )
 984                 elif stations.has_section(place):
 985                     station = (place, 0.0)
 986                     description = stations.get(place, "description")
 987                     if stations.has_option(place, "zone"):
 988                         zone = eval( stations.get(place, "zone" ) )
 989                     search = ( expression, "ICAO station code %s" % place )
 990                 elif zones.has_section(place):
 991                     station = eval( zones.get(place, "station") )
 992                     description = zones.get(place, "description")
 993                     zone = (place, 0.0)
 994                     search = ( expression, "NWS/NOAA weather zone %s" % place )
 995                 if not ( info or quiet ):
 996                     print( "[%s result %s]" % (action, description) )
 997             if not possibilities and not station[0]:
 998                 message = "No FIPS code/census area match in the %s file.\n" % (
 999                     datafiles["places"][0]
1000                 )
1001                 sys.stderr.write(message)
1002                 exit(1)
1003     if station[0]:
1004         uris["metar"] = stations.get( station[0], "metar" )
1005         if zone[0]:
1006             for key,value in zones.items( zone[0] ):
1007                 if key not in ("centroid", "description", "station"):
1008                     uris[key] = value
1009     elif possibilities:
1010         count = len(possibilities)
1011         if count <= max_results:
1012             print( "Your search is ambiguous, returning %s matches:" % count )
1013             for place in sorted(possibilities):
1014                 if places.has_section(place):
1015                     print(
1016                         "   [%s] %s" % (
1017                             place,
1018                             places.get(place, "description")
1019                         )
1020                     )
1021                 elif stations.has_section(place):
1022                     print(
1023                         "   [%s] %s" % (
1024                             place,
1025                             stations.get(place, "description")
1026                         )
1027                     )
1028                 elif zones.has_section(place):
1029                     print(
1030                         "   [%s] %s" % (
1031                             place,
1032                             zones.get(place, "description")
1033                         )
1034                     )
1035         else:
1036             print(
1037                 "Your search is too ambiguous, returning %s matches." % count
1038             )
1039         exit(0)
1040     if info:
1041         stationlist = []
1042         zonelist = []
1043         if dataset:
1044             for section in dataset.sections():
1045                 if dataset.has_option(section, "station"):
1046                     stationlist.append(
1047                         eval( dataset.get(section, "station") )[1]
1048                     )
1049                 if dataset.has_option(section, "zone"):
1050                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1051         stationlist.sort()
1052         zonelist.sort()
1053         scount = len(stationlist)
1054         zcount = len(zonelist)
1055         sranks = []
1056         zranks = []
1057         for score in scores:
1058             if stationlist:
1059                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1060             if zonelist:
1061                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1062         description = search[1]
1063         uris["description"] = description
1064         print(
1065             "%s\n%s" % ( description, "-" * len(description) )
1066         )
1067         print(
1068             "%s: %s" % (
1069                 station[0],
1070                 stations.get( station[0], "description" )
1071             )
1072         )
1073         km = radian_to_km*station[1]
1074         mi = radian_to_mi*station[1]
1075         if sranks and not description.startswith("ICAO station code "):
1076             for index in range(0, len(scores)):
1077                 if station[1] >= sranks[index]:
1078                     score = scores[index][1]
1079                     break
1080             print(
1081                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1082             )
1083         elif searchtype == "coordinates":
1084             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1085         if zone[0]:
1086             print(
1087                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1088             )
1089         km = radian_to_km*zone[1]
1090         mi = radian_to_mi*zone[1]
1091         if zranks and not description.startswith("NWS/NOAA weather zone "):
1092             for index in range(0, len(scores)):
1093                 if zone[1] >= zranks[index]:
1094                     score = scores[index][1]
1095                     break
1096             print(
1097                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1098             )
1099         elif searchtype == "coordinates" and zone[0]:
1100             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1101     if cache_search:
1102         now = time.time()
1103         nowstamp = "%s (%s)" % (
1104             now,
1105             datetime.datetime.isoformat(
1106                 datetime.datetime.fromtimestamp(now),
1107                 " "
1108             )
1109         )
1110         search_cache = ["\n"]
1111         search_cache.append( "[%s]\n" % search[0] )
1112         search_cache.append( "cached = %s\n" % nowstamp )
1113         for uriname in sorted(uris.keys()):
1114             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1115         real_cachedir = os.path.expanduser(cachedir)
1116         if not os.path.exists(real_cachedir):
1117             try: os.makedirs(real_cachedir)
1118             except (IOError, OSError): pass
1119         scache_fn = os.path.join(real_cachedir, "searches")
1120         if not os.path.exists(scache_fn):
1121             then = sorted(
1122                     [ x[1] for x in datafiles.values() ],
1123                     reverse=True
1124                 )[0]
1125             thenstamp = "%s (%s)" % (
1126                 then,
1127                 datetime.datetime.isoformat(
1128                     datetime.datetime.fromtimestamp(then),
1129                     " "
1130                 )
1131             )
1132             search_cache.insert(
1133                 0,
1134                 "# based on data files from: %s\n" % thenstamp
1135             )
1136         try:
1137             scache_existing = configparser.ConfigParser()
1138             scache_existing.read(scache_fn, encoding="utf-8")
1139             if not scache_existing.has_section(search[0]):
1140                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1141                 scache_fd.writelines(search_cache)
1142                 scache_fd.close()
1143         except (IOError, OSError): pass
1144     if not info:
1145         return(uris)
1146
1147 def closest(position, nodes, fieldname, angle=None):
1148     import math
1149     if not angle: angle = 2*math.pi
1150     match = None
1151     for name in nodes:
1152         if fieldname in nodes[name]:
1153             node = nodes[name][fieldname]
1154             if node and abs( position[0]-node[0] ) < angle:
1155                 if abs( position[1]-node[1] ) < angle \
1156                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1157                     if position == node:
1158                         angle = 0
1159                         match = name
1160                     else:
1161                         candidate = math.acos(
1162                             math.sin( position[0] ) * math.sin( node[0] ) \
1163                                 + math.cos( position[0] ) \
1164                                 * math.cos( node[0] ) \
1165                                 * math.cos( position[1] - node[1] )
1166                             )
1167                         if candidate < angle:
1168                             angle = candidate
1169                             match = name
1170     if match: match = str(match)
1171     return (match, angle)
1172
1173 def gecos(formatted):
1174     import math, re
1175     coordinates = formatted.split(",")
1176     for coordinate in range(0, 2):
1177         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1178             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1179             coordinates[coordinate].strip().lower()
1180         ).groups()
1181         value = float(degrees)
1182         if minutes: value += float(minutes)/60
1183         if seconds: value += float(seconds)/3600
1184         if hemisphere and hemisphere in "sw": value *= -1
1185         coordinates[coordinate] = math.radians(value)
1186     return tuple(coordinates)
1187
1188 def correlate():
1189     import codecs, configparser, csv, datetime, hashlib, os, re, sys, time
1190     import zipfile, zoneinfo
1191     for filename in os.listdir("."):
1192         if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename):
1193             gcounties_an = filename
1194             gcounties_fn = filename[:-4] + ".txt"
1195         elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename):
1196             gcousubs_an = filename
1197             gcousubs_fn = filename[:-4] + ".txt"
1198         elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename):
1199             gplace_an = filename
1200             gplace_fn = filename[:-4] + ".txt"
1201         elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename):
1202             gzcta_an = filename
1203             gzcta_fn = filename[:-4] + ".txt"
1204         elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename):
1205             cpfzcf_fn = filename
1206     nsdcccc_fn = "nsd_cccc.txt"
1207     ourairports_fn = "airports.csv"
1208     overrides_fn = "overrides.conf"
1209     overrideslog_fn = "overrides.log"
1210     slist_fn = "slist"
1211     zlist_fn = "zlist"
1212     qalog_fn = "qa.log"
1213     airports_fn = "airports"
1214     places_fn = "places"
1215     stations_fn = "stations"
1216     zctas_fn = "zctas"
1217     zones_fn = "zones"
1218     header = """\
1219 %s
1220 # generated by %s on %s from these public domain sources:
1221 #
1222 # https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html
1223 # %s %s %s
1224 # %s %s %s
1225 # %s %s %s
1226 # %s %s %s
1227 #
1228 # https://www.weather.gov/gis/ZoneCounty/
1229 # %s %s %s
1230 #
1231 # https://tgftp.nws.noaa.gov/data/
1232 # %s %s %s
1233 #
1234 # https://ourairports.com/data/
1235 # %s %s %s
1236 #
1237 # ...and these manually-generated or hand-compiled adjustments:
1238 # %s %s %s
1239 # %s %s %s
1240 # %s %s %s\
1241 """ % (
1242         weather_copyright,
1243         os.path.basename( sys.argv[0] ),
1244         datetime.date.isoformat(
1245             datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
1246         ),
1247         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1248         datetime.date.isoformat(
1249             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
1250         ),
1251         gcounties_an,
1252         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1253         datetime.date.isoformat(
1254             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
1255         ),
1256         gcousubs_an,
1257         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
1258         datetime.date.isoformat(
1259             datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
1260         ),
1261         gplace_an,
1262         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1263         datetime.date.isoformat(
1264             datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
1265         ),
1266         gzcta_an,
1267         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1268         datetime.date.isoformat(
1269             datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
1270         ),
1271         cpfzcf_fn,
1272         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1273         datetime.date.isoformat(
1274             datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
1275         ),
1276         nsdcccc_fn,
1277         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
1278         datetime.date.isoformat(
1279             datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
1280         ),
1281         ourairports_fn,
1282         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1283         datetime.date.isoformat(
1284             datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
1285         ),
1286         overrides_fn,
1287         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1288         datetime.date.isoformat(
1289             datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
1290         ),
1291         slist_fn,
1292         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1293         datetime.date.isoformat(
1294             datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
1295         ),
1296         zlist_fn
1297     )
1298     airports = {}
1299     places = {}
1300     stations = {}
1301     zctas = {}
1302     zones = {}
1303     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1304     sys.stdout.write(message)
1305     sys.stdout.flush()
1306     count = 0
1307     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
1308     columns = gcounties.readline().decode("utf-8").strip().split("\t")
1309     for line in gcounties:
1310         fields = line.decode("utf-8").strip().split("\t")
1311         f_geoid = fields[ columns.index("GEOID") ].strip()
1312         f_name = fields[ columns.index("NAME") ].strip()
1313         f_usps = fields[ columns.index("USPS") ].strip()
1314         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1315         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1316         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1317             fips = "fips%s" % f_geoid
1318             if fips not in places: places[fips] = {}
1319             places[fips]["centroid"] = gecos(
1320                 "%s,%s" % (f_intptlat, f_intptlong)
1321             )
1322             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1323             count += 1
1324     gcounties.close()
1325     print("done (%s lines)." % count)
1326     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1327     sys.stdout.write(message)
1328     sys.stdout.flush()
1329     count = 0
1330     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
1331     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
1332     for line in gcousubs:
1333         fields = line.decode("utf-8").strip().split("\t")
1334         f_geoid = fields[ columns.index("GEOID") ].strip()
1335         f_name = fields[ columns.index("NAME") ].strip()
1336         f_usps = fields[ columns.index("USPS") ].strip()
1337         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1338         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1339         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1340             fips = "fips%s" % f_geoid
1341             if fips not in places: places[fips] = {}
1342             places[fips]["centroid"] = gecos(
1343                 "%s,%s" % (f_intptlat, f_intptlong)
1344             )
1345             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1346             count += 1
1347     gcousubs.close()
1348     print("done (%s lines)." % count)
1349     message = "Reading %s:%s..." % (gplace_an, gplace_fn)
1350     sys.stdout.write(message)
1351     sys.stdout.flush()
1352     count = 0
1353     gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
1354     columns = gplace.readline().decode("utf-8").strip().split("\t")
1355     for line in gplace:
1356         fields = line.decode("utf-8").strip().split("\t")
1357         f_geoid = fields[ columns.index("GEOID") ].strip()
1358         f_name = fields[ columns.index("NAME") ].strip()
1359         f_usps = fields[ columns.index("USPS") ].strip()
1360         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1361         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1362         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1363             fips = "fips%s" % f_geoid
1364             if fips not in places: places[fips] = {}
1365             places[fips]["centroid"] = gecos(
1366                 "%s,%s" % (f_intptlat, f_intptlong)
1367             )
1368             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1369             count += 1
1370     gplace.close()
1371     print("done (%s lines)." % count)
1372     message = "Reading %s..." % slist_fn
1373     sys.stdout.write(message)
1374     sys.stdout.flush()
1375     count = 0
1376     slist = codecs.open(slist_fn, "r", "utf-8")
1377     for line in slist:
1378         icao = line.split("#")[0].strip()
1379         if icao:
1380             stations[icao] = {
1381                 "metar": "https://tgftp.nws.noaa.gov/data/observations/"\
1382                     + "metar/decoded/%s.TXT" % icao.upper()
1383             }
1384             count += 1
1385     slist.close()
1386     print("done (%s lines)." % count)
1387     message = "Reading %s..." % nsdcccc_fn
1388     sys.stdout.write(message)
1389     sys.stdout.flush()
1390     count = 0
1391     nsdcccc = codecs.open(nsdcccc_fn, "r", "utf-8")
1392     for line in nsdcccc:
1393         line = str(line)
1394         fields = line.split(";")
1395         icao = fields[0].strip().lower()
1396         if icao in stations:
1397             description = []
1398             name = " ".join( fields[3].strip().title().split() )
1399             if name: description.append(name)
1400             st = fields[4].strip()
1401             if st: description.append(st)
1402             country = " ".join( fields[5].strip().title().split() )
1403             if country: description.append(country)
1404             if description:
1405                 stations[icao]["description"] = ", ".join(description)
1406             lat, lon = fields[7:9]
1407             if lat and lon:
1408                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1409             elif "location" not in stations[icao]:
1410                 lat, lon = fields[5:7]
1411                 if lat and lon:
1412                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1413         count += 1
1414     nsdcccc.close()
1415     print("done (%s lines)." % count)
1416     message = "Reading %s..." % ourairports_fn
1417     sys.stdout.write(message)
1418     sys.stdout.flush()
1419     count = 0
1420     ourairports = open(ourairports_fn, "r")
1421     for row in csv.reader(ourairports):
1422         icao = row[12].lower()
1423         if icao in stations:
1424             iata = row[13].lower()
1425             if len(iata) == 3: airports[iata] = { "station": icao }
1426             if "description" not in stations[icao]:
1427                 description = []
1428                 name = row[3]
1429                 if name: description.append(name)
1430                 municipality = row[10]
1431                 if municipality: description.append(municipality)
1432                 region = row[9]
1433                 country = row[8]
1434                 if region:
1435                     if "-" in region:
1436                         c,r = region.split("-", 1)
1437                         if c == country: region = r
1438                     description.append(region)
1439                 if country:
1440                     description.append(country)
1441                 if description:
1442                     stations[icao]["description"] = ", ".join(description)
1443             if "location" not in stations[icao]:
1444                 lat = row[4]
1445                 if lat:
1446                     lon = row[5]
1447                     if lon:
1448                         stations[icao]["location"] = gecos(
1449                             "%s,%s" % (lat, lon)
1450                         )
1451         count += 1
1452     ourairports.close()
1453     print("done (%s lines)." % count)
1454     message = "Reading %s..." % zlist_fn
1455     sys.stdout.write(message)
1456     sys.stdout.flush()
1457     count = 0
1458     zlist = codecs.open(zlist_fn, "r", "utf-8")
1459     for line in zlist:
1460         line = line.split("#")[0].strip()
1461         if line:
1462             zones[line] = {}
1463             count += 1
1464     zlist.close()
1465     print("done (%s lines)." % count)
1466     message = "Reading %s..." % cpfzcf_fn
1467     sys.stdout.write(message)
1468     sys.stdout.flush()
1469     count = 0
1470     cpfz = {}
1471     cpfzcf = codecs.open(cpfzcf_fn, "r", "utf-8")
1472     for line in cpfzcf:
1473         fields = line.strip().split("|")
1474         if len(fields) == 11 \
1475             and fields[0] and fields[1] and fields[9] and fields[10]:
1476             zone = "z".join( fields[:2] ).lower()
1477             if zone in zones:
1478                 state = fields[0]
1479                 description = fields[3].strip()
1480                 fips = "fips%s"%fields[6]
1481                 countycode = "%sc%s" % (state.lower(), fips[-3:])
1482                 if state:
1483                     zones[zone]["coastal_flood_statement"] = (
1484                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1485                         "flood/coastal/%s/%s.txt" % (state.lower(), zone))
1486                     zones[zone]["flash_flood_statement"] = (
1487                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1488                         "flash_flood/statement/%s/%s.txt"
1489                         % (state.lower(), countycode))
1490                     zones[zone]["flash_flood_warning"] = (
1491                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1492                         "flash_flood/warning/%s/%s.txt"
1493                         % (state.lower(), countycode))
1494                     zones[zone]["flash_flood_watch"] = (
1495                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1496                         "flash_flood/watch/%s/%s.txt" % (state.lower(), zone))
1497                     zones[zone]["flood_warning"] = (
1498                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1499                         "flood/warning/%s/%s.txt"
1500                         % (state.lower(), countycode))
1501                     zones[zone]["severe_thunderstorm_warning"] = (
1502                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1503                         "thunderstorm/%s/%s.txt" % (state.lower(), countycode))
1504                     zones[zone]["severe_weather_statement"] = (
1505                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1506                         "severe_weather_stmt/%s/%s.txt"
1507                         % (state.lower(), countycode))
1508                     zones[zone]["short_term_forecast"] = (
1509                         "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/"
1510                         "%s/%s.txt" % (state.lower(), zone))
1511                     zones[zone]["special_weather_statement"] = (
1512                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1513                         "special_weather_stmt/%s/%s.txt"
1514                         % (state.lower(), zone))
1515                     zones[zone]["state_forecast"] = (
1516                         "https://tgftp.nws.noaa.gov/data/forecasts/state/"
1517                         "%s/%s.txt" % (state.lower(), zone))
1518                     zones[zone]["tornado"] = (
1519                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1520                         "tornado/%s/%s.txt" % (state.lower(), countycode))
1521                     zones[zone]["urgent_weather_message"] = (
1522                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1523                         "non_precip/%s/%s.txt" % (state.lower(), zone))
1524                     zones[zone]["zone_forecast"] = (
1525                         "https://tgftp.nws.noaa.gov/data/forecasts/zone/"
1526                         "%s/%s.txt" % (state.lower(), zone))
1527                 tzcode = fields[7]
1528                 if tzcode == "A":
1529                     zones[zone]["tz"] = "US/Alaska"
1530                 elif tzcode == "AH":
1531                     zones[zone]["tz"] = "US/Aleutian"
1532                 elif tzcode in ("C", "CE", "CM"):
1533                     zones[zone]["tz"] = "US/Central"
1534                 elif tzcode in ("E", "e"):
1535                     zones[zone]["tz"] = "US/Eastern"
1536                 elif tzcode == "F":
1537                     zones[zone]["tz"] = "Pacific/Guadalcanal"
1538                 elif tzcode == "G":
1539                     zones[zone]["tz"] = "Pacific/Guam"
1540                 elif tzcode == "H":
1541                     zones[zone]["tz"] = "US/Hawaii"
1542                 elif tzcode == "J":
1543                     zones[zone]["tz"] = "Japan"
1544                 elif tzcode == "K":
1545                     zones[zone]["tz"] = "Pacific/Kwajalein"
1546                 elif tzcode in ("M", "MC", "MP"):
1547                     zones[zone]["tz"] = "US/Mountain"
1548                 elif tzcode == "m":
1549                     zones[zone]["tz"] = "US/Arizona"
1550                 elif tzcode == "P":
1551                     zones[zone]["tz"] = "US/Pacific"
1552                 elif tzcode == "S":
1553                     zones[zone]["tz"] = "US/Samoa"
1554                 elif tzcode == "V":
1555                     zones[zone]["tz"] = "America/Virgin"
1556                 else:
1557                     zones[zone]["tz"] = ""
1558                 county = fields[5]
1559                 if county:
1560                     if description.endswith(county):
1561                         description += " County"
1562                     else:
1563                         description += ", %s County" % county
1564                 description += ", %s, US" % state
1565                 zones[zone]["description"] = description
1566                 zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
1567                 if fips in places and not zones[zone]["centroid"]:
1568                     zones[zone]["centroid"] = places[fips]["centroid"]
1569         count += 1
1570     cpfzcf.close()
1571     print("done (%s lines)." % count)
1572     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1573     sys.stdout.write(message)
1574     sys.stdout.flush()
1575     count = 0
1576     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
1577     columns = gzcta.readline().decode("utf-8").strip().split("\t")
1578     for line in gzcta:
1579         fields = line.decode("utf-8").strip().split("\t")
1580         f_geoid = fields[ columns.index("GEOID") ].strip()
1581         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1582         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1583         if f_geoid and f_intptlat and f_intptlong:
1584             if f_geoid not in zctas: zctas[f_geoid] = {}
1585             zctas[f_geoid]["centroid"] = gecos(
1586                 "%s,%s" % (f_intptlat, f_intptlong)
1587             )
1588             count += 1
1589     gzcta.close()
1590     print("done (%s lines)." % count)
1591     message = "Reading %s..." % overrides_fn
1592     sys.stdout.write(message)
1593     sys.stdout.flush()
1594     count = 0
1595     added = 0
1596     removed = 0
1597     changed = 0
1598     overrides = configparser.ConfigParser()
1599     overrides.read_file( codecs.open(overrides_fn, "r", "utf8") )
1600     overrideslog = []
1601     for section in overrides.sections():
1602         addopt = 0
1603         chgopt = 0
1604         if section.startswith("-"):
1605             section = section[1:]
1606             delete = True
1607         else: delete = False
1608         if re.match("[A-Za-z]{3}$", section):
1609             if delete:
1610                 if section in airports:
1611                     del( airports[section] )
1612                     logact = "removed airport %s" % section
1613                     removed += 1
1614                 else:
1615                     logact = "tried to remove nonexistent airport %s" % section
1616             else:
1617                 if section in airports:
1618                     logact = "changed airport %s" % section
1619                     changed += 1
1620                 else:
1621                     airports[section] = {}
1622                     logact = "added airport %s" % section
1623                     added += 1
1624                 for key,value in overrides.items(section):
1625                     if key in airports[section]: chgopt += 1
1626                     else: addopt += 1
1627                     if key in ("centroid", "location"):
1628                         airports[section][key] = eval(value)
1629                     else:
1630                         airports[section][key] = value
1631                 if addopt and chgopt:
1632                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1633                 elif addopt: logact += " (+%s options)" % addopt
1634                 elif chgopt: logact += " (!%s options)" % chgopt
1635         elif re.match("[A-Za-z0-9]{4}$", section):
1636             if delete:
1637                 if section in stations:
1638                     del( stations[section] )
1639                     logact = "removed station %s" % section
1640                     removed += 1
1641                 else:
1642                     logact = "tried to remove nonexistent station %s" % section
1643             else:
1644                 if section in stations:
1645                     logact = "changed station %s" % section
1646                     changed += 1
1647                 else:
1648                     stations[section] = {}
1649                     logact = "added station %s" % section
1650                     added += 1
1651                 for key,value in overrides.items(section):
1652                     if key in stations[section]: chgopt += 1
1653                     else: addopt += 1
1654                     if key in ("centroid", "location"):
1655                         stations[section][key] = eval(value)
1656                     else:
1657                         stations[section][key] = value
1658                 if addopt and chgopt:
1659                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1660                 elif addopt: logact += " (+%s options)" % addopt
1661                 elif chgopt: logact += " (!%s options)" % chgopt
1662         elif re.match("[0-9]{5}$", section):
1663             if delete:
1664                 if section in zctas:
1665                     del( zctas[section] )
1666                     logact = "removed zcta %s" % section
1667                     removed += 1
1668                 else:
1669                     logact = "tried to remove nonexistent zcta %s" % section
1670             else:
1671                 if section in zctas:
1672                     logact = "changed zcta %s" % section
1673                     changed += 1
1674                 else:
1675                     zctas[section] = {}
1676                     logact = "added zcta %s" % section
1677                     added += 1
1678                 for key,value in overrides.items(section):
1679                     if key in zctas[section]: chgopt += 1
1680                     else: addopt += 1
1681                     if key in ("centroid", "location"):
1682                         zctas[section][key] = eval(value)
1683                     else:
1684                         zctas[section][key] = value
1685                 if addopt and chgopt:
1686                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1687                 elif addopt: logact += " (+%s options)" % addopt
1688                 elif chgopt: logact += " (!%s options)" % chgopt
1689         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1690             if delete:
1691                 if section in zones:
1692                     del( zones[section] )
1693                     logact = "removed zone %s" % section
1694                     removed += 1
1695                 else:
1696                     logact = "tried to remove nonexistent zone %s" % section
1697             else:
1698                 if section in zones:
1699                     logact = "changed zone %s" % section
1700                     changed += 1
1701                 else:
1702                     zones[section] = {}
1703                     logact = "added zone %s" % section
1704                     added += 1
1705                 for key,value in overrides.items(section):
1706                     if key in zones[section]: chgopt += 1
1707                     else: addopt += 1
1708                     if key in ("centroid", "location"):
1709                         zones[section][key] = eval(value)
1710                     else:
1711                         zones[section][key] = value
1712                 if addopt and chgopt:
1713                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1714                 elif addopt: logact += " (+%s options)" % addopt
1715                 elif chgopt: logact += " (!%s options)" % chgopt
1716         elif re.match("fips[0-9]+$", section):
1717             if delete:
1718                 if section in places:
1719                     del( places[section] )
1720                     logact = "removed place %s" % section
1721                     removed += 1
1722                 else:
1723                     logact = "tried to remove nonexistent place %s" % section
1724             else:
1725                 if section in places:
1726                     logact = "changed place %s" % section
1727                     changed += 1
1728                 else:
1729                     places[section] = {}
1730                     logact = "added place %s" % section
1731                     added += 1
1732                 for key,value in overrides.items(section):
1733                     if key in places[section]: chgopt += 1
1734                     else: addopt += 1
1735                     if key in ("centroid", "location"):
1736                         places[section][key] = eval(value)
1737                     else:
1738                         places[section][key] = value
1739                 if addopt and chgopt:
1740                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1741                 elif addopt: logact += " (+%s options)" % addopt
1742                 elif chgopt: logact += " (!%s options)" % chgopt
1743         count += 1
1744         overrideslog.append("%s\n" % logact)
1745     overrideslog.sort()
1746     if os.path.exists(overrideslog_fn):
1747         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1748     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1749     import time
1750     overrideslog_fd.write(
1751         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
1752         '# use, copy, modify, and distribute this software is granted under terms\n'
1753         '# provided in the LICENSE file distributed with this software.\n\n'
1754         % time.gmtime().tm_year)
1755     overrideslog_fd.writelines(overrideslog)
1756     overrideslog_fd.close()
1757     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1758         count,
1759         added,
1760         removed,
1761         changed
1762     ) )
1763     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1764     print(
1765         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1766             estimate
1767     )
1768     count = 0
1769     milestones = list( range(51) )
1770     message = "   "
1771     sys.stdout.write(message)
1772     sys.stdout.flush()
1773     for fips in places:
1774         centroid = places[fips]["centroid"]
1775         if centroid:
1776             station = closest(centroid, stations, "location", 0.1)
1777         if station[0]:
1778             places[fips]["station"] = station
1779             count += 1
1780             if not count%100:
1781                 level = int(50*count/estimate)
1782                 if level in milestones:
1783                     for remaining in milestones[:milestones.index(level)+1]:
1784                         if remaining%5:
1785                             message = "."
1786                             sys.stdout.write(message)
1787                             sys.stdout.flush()
1788                         else:
1789                             message = "%s%%" % (remaining*2,)
1790                             sys.stdout.write(message)
1791                             sys.stdout.flush()
1792                         milestones.remove(remaining)
1793         if centroid:
1794             zone = closest(centroid, zones, "centroid", 0.1)
1795         if zone[0]:
1796             places[fips]["zone"] = zone
1797             count += 1
1798             if not count%100:
1799                 level = int(50*count/estimate)
1800                 if level in milestones:
1801                     for remaining in milestones[:milestones.index(level)+1]:
1802                         if remaining%5:
1803                             message = "."
1804                             sys.stdout.write(message)
1805                             sys.stdout.flush()
1806                         else:
1807                             message = "%s%%" % (remaining*2,)
1808                             sys.stdout.write(message)
1809                             sys.stdout.flush()
1810                         milestones.remove(remaining)
1811     for station in stations:
1812         if "location" in stations[station]:
1813             location = stations[station]["location"]
1814             if location:
1815                 zone = closest(location, zones, "centroid", 0.1)
1816             if zone[0]:
1817                 stations[station]["zone"] = zone
1818                 count += 1
1819                 if not count%100:
1820                     level = int(50*count/estimate)
1821                     if level in milestones:
1822                         for remaining in milestones[:milestones.index(level)+1]:
1823                             if remaining%5:
1824                                 message = "."
1825                                 sys.stdout.write(message)
1826                                 sys.stdout.flush()
1827                             else:
1828                                 message = "%s%%" % (remaining*2,)
1829                                 sys.stdout.write(message)
1830                                 sys.stdout.flush()
1831                             milestones.remove(remaining)
1832     for zcta in zctas.keys():
1833         centroid = zctas[zcta]["centroid"]
1834         if centroid:
1835             station = closest(centroid, stations, "location", 0.1)
1836         if station[0]:
1837             zctas[zcta]["station"] = station
1838             count += 1
1839             if not count%100:
1840                 level = int(50*count/estimate)
1841                 if level in milestones:
1842                     for remaining in milestones[ : milestones.index(level)+1 ]:
1843                         if remaining%5:
1844                             message = "."
1845                             sys.stdout.write(message)
1846                             sys.stdout.flush()
1847                         else:
1848                             message = "%s%%" % (remaining*2,)
1849                             sys.stdout.write(message)
1850                             sys.stdout.flush()
1851                         milestones.remove(remaining)
1852         if centroid:
1853             zone = closest(centroid, zones, "centroid", 0.1)
1854         if zone[0]:
1855             zctas[zcta]["zone"] = zone
1856             count += 1
1857             if not count%100:
1858                 level = int(50*count/estimate)
1859                 if level in milestones:
1860                     for remaining in milestones[:milestones.index(level)+1]:
1861                         if remaining%5:
1862                             message = "."
1863                             sys.stdout.write(message)
1864                             sys.stdout.flush()
1865                         else:
1866                             message = "%s%%" % (remaining*2,)
1867                             sys.stdout.write(message)
1868                             sys.stdout.flush()
1869                         milestones.remove(remaining)
1870     for zone in zones.keys():
1871         if "centroid" in zones[zone]:
1872             centroid = zones[zone]["centroid"]
1873             if centroid:
1874                 station = closest(centroid, stations, "location", 0.1)
1875             if station[0]:
1876                 zones[zone]["station"] = station
1877                 count += 1
1878                 if not count%100:
1879                     level = int(50*count/estimate)
1880                     if level in milestones:
1881                         for remaining in milestones[:milestones.index(level)+1]:
1882                             if remaining%5:
1883                                 message = "."
1884                                 sys.stdout.write(message)
1885                                 sys.stdout.flush()
1886                             else:
1887                                 message = "%s%%" % (remaining*2,)
1888                                 sys.stdout.write(message)
1889                                 sys.stdout.flush()
1890                             milestones.remove(remaining)
1891     for remaining in milestones:
1892         if remaining%5:
1893             message = "."
1894             sys.stdout.write(message)
1895             sys.stdout.flush()
1896         else:
1897             message = "%s%%" % (remaining*2,)
1898             sys.stdout.write(message)
1899             sys.stdout.flush()
1900     print("\n   done (%s correlations)." % count)
1901     message = "Writing %s..." % airports_fn
1902     sys.stdout.write(message)
1903     sys.stdout.flush()
1904     count = 0
1905     if os.path.exists(airports_fn):
1906         os.rename(airports_fn, "%s_old"%airports_fn)
1907     airports_fd = codecs.open(airports_fn, "w", "utf8")
1908     airports_fd.write(header)
1909     for airport in sorted( airports.keys() ):
1910         airports_fd.write("\n\n[%s]" % airport)
1911         for key, value in sorted( airports[airport].items() ):
1912             if type(value) is float: value = "%.7f"%value
1913             elif type(value) is tuple:
1914                 elements = []
1915                 for element in value:
1916                     if type(element) is float: elements.append("%.7f"%element)
1917                     else: elements.append( repr(element) )
1918                 value = "(%s)"%", ".join(elements)
1919             airports_fd.write( "\n%s = %s" % (key, value) )
1920         count += 1
1921     airports_fd.write("\n")
1922     airports_fd.close()
1923     print("done (%s sections)." % count)
1924     message = "Writing %s..." % places_fn
1925     sys.stdout.write(message)
1926     sys.stdout.flush()
1927     count = 0
1928     if os.path.exists(places_fn):
1929         os.rename(places_fn, "%s_old"%places_fn)
1930     places_fd = codecs.open(places_fn, "w", "utf8")
1931     places_fd.write(header)
1932     for fips in sorted( places.keys() ):
1933         places_fd.write("\n\n[%s]" % fips)
1934         for key, value in sorted( places[fips].items() ):
1935             if type(value) is float: value = "%.7f"%value
1936             elif type(value) is tuple:
1937                 elements = []
1938                 for element in value:
1939                     if type(element) is float: elements.append("%.7f"%element)
1940                     else: elements.append( repr(element) )
1941                 value = "(%s)"%", ".join(elements)
1942             places_fd.write( "\n%s = %s" % (key, value) )
1943         count += 1
1944     places_fd.write("\n")
1945     places_fd.close()
1946     print("done (%s sections)." % count)
1947     message = "Writing %s..." % stations_fn
1948     sys.stdout.write(message)
1949     sys.stdout.flush()
1950     count = 0
1951     if os.path.exists(stations_fn):
1952         os.rename(stations_fn, "%s_old"%stations_fn)
1953     stations_fd = codecs.open(stations_fn, "w", "utf-8")
1954     stations_fd.write(header)
1955     for station in sorted( stations.keys() ):
1956         stations_fd.write("\n\n[%s]" % station)
1957         for key, value in sorted( stations[station].items() ):
1958             if type(value) is float: value = "%.7f"%value
1959             elif type(value) is tuple:
1960                 elements = []
1961                 for element in value:
1962                     if type(element) is float: elements.append("%.7f"%element)
1963                     else: elements.append( repr(element) )
1964                 value = "(%s)"%", ".join(elements)
1965             if type(value) is bytes:
1966                 value = value.decode("utf-8")
1967             stations_fd.write( "\n%s = %s" % (key, value) )
1968         count += 1
1969     stations_fd.write("\n")
1970     stations_fd.close()
1971     print("done (%s sections)." % count)
1972     message = "Writing %s..." % zctas_fn
1973     sys.stdout.write(message)
1974     sys.stdout.flush()
1975     count = 0
1976     if os.path.exists(zctas_fn):
1977         os.rename(zctas_fn, "%s_old"%zctas_fn)
1978     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
1979     zctas_fd.write(header)
1980     for zcta in sorted( zctas.keys() ):
1981         zctas_fd.write("\n\n[%s]" % zcta)
1982         for key, value in sorted( zctas[zcta].items() ):
1983             if type(value) is float: value = "%.7f"%value
1984             elif type(value) is tuple:
1985                 elements = []
1986                 for element in value:
1987                     if type(element) is float: elements.append("%.7f"%element)
1988                     else: elements.append( repr(element) )
1989                 value = "(%s)"%", ".join(elements)
1990             zctas_fd.write( "\n%s = %s" % (key, value) )
1991         count += 1
1992     zctas_fd.write("\n")
1993     zctas_fd.close()
1994     print("done (%s sections)." % count)
1995     message = "Writing %s..." % zones_fn
1996     sys.stdout.write(message)
1997     sys.stdout.flush()
1998     count = 0
1999     if os.path.exists(zones_fn):
2000         os.rename(zones_fn, "%s_old"%zones_fn)
2001     zones_fd = codecs.open(zones_fn, "w", "utf8")
2002     zones_fd.write(header)
2003     for zone in sorted( zones.keys() ):
2004         zones_fd.write("\n\n[%s]" % zone)
2005         for key, value in sorted( zones[zone].items() ):
2006             if type(value) is float: value = "%.7f"%value
2007             elif type(value) is tuple:
2008                 elements = []
2009                 for element in value:
2010                     if type(element) is float: elements.append("%.7f"%element)
2011                     else: elements.append( repr(element) )
2012                 value = "(%s)"%", ".join(elements)
2013             zones_fd.write( "\n%s = %s" % (key, value) )
2014         count += 1
2015     zones_fd.write("\n")
2016     zones_fd.close()
2017     print("done (%s sections)." % count)
2018     message = "Starting QA check..."
2019     sys.stdout.write(message)
2020     sys.stdout.flush()
2021     airports = configparser.ConfigParser()
2022     airports.read(airports_fn, encoding="utf-8")
2023     places = configparser.ConfigParser()
2024     places.read(places_fn, encoding="utf-8")
2025     stations = configparser.ConfigParser()
2026     stations.read(stations_fn, encoding="utf-8")
2027     zctas = configparser.ConfigParser()
2028     zctas.read(zctas_fn, encoding="utf-8")
2029     zones = configparser.ConfigParser()
2030     zones.read(zones_fn, encoding="utf-8")
2031     qalog = []
2032     places_nocentroid = 0
2033     places_nodescription = 0
2034     for place in sorted( places.sections() ):
2035         if not places.has_option(place, "centroid"):
2036             qalog.append("%s: no centroid\n" % place)
2037             places_nocentroid += 1
2038         if not places.has_option(place, "description"):
2039             qalog.append("%s: no description\n" % place)
2040             places_nodescription += 1
2041     stations_nodescription = 0
2042     stations_nolocation = 0
2043     stations_nometar = 0
2044     for station in sorted( stations.sections() ):
2045         if not stations.has_option(station, "description"):
2046             qalog.append("%s: no description\n" % station)
2047             stations_nodescription += 1
2048         if not stations.has_option(station, "location"):
2049             qalog.append("%s: no location\n" % station)
2050             stations_nolocation += 1
2051         if not stations.has_option(station, "metar"):
2052             qalog.append("%s: no metar\n" % station)
2053             stations_nometar += 1
2054     airports_badstation = 0
2055     airports_nostation = 0
2056     for airport in sorted( airports.sections() ):
2057         if not airports.has_option(airport, "station"):
2058             qalog.append("%s: no station\n" % airport)
2059             airports_nostation += 1
2060         else:
2061             station = airports.get(airport, "station")
2062             if station not in stations.sections():
2063                 qalog.append( "%s: bad station %s\n" % (airport, station) )
2064                 airports_badstation += 1
2065     zctas_nocentroid = 0
2066     for zcta in sorted( zctas.sections() ):
2067         if not zctas.has_option(zcta, "centroid"):
2068             qalog.append("%s: no centroid\n" % zcta)
2069             zctas_nocentroid += 1
2070     zones_nocentroid = 0
2071     zones_nodescription = 0
2072     zones_notz = 0
2073     zones_noforecast = 0
2074     zones_overlapping = 0
2075     zonetable = {}
2076     for zone in zones.sections():
2077         if zones.has_option(zone, "centroid"):
2078             zonetable[zone] = {
2079                 "centroid": eval( zones.get(zone, "centroid") )
2080             }
2081     for zone in sorted( zones.sections() ):
2082         if zones.has_option(zone, "centroid"):
2083             zonetable_local = zonetable.copy()
2084             del( zonetable_local[zone] )
2085             centroid = eval( zones.get(zone, "centroid") )
2086             if centroid:
2087                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2088             if nearest[1]*radian_to_km < 1:
2089                 qalog.append( "%s: within one km of %s\n" % (
2090                     zone,
2091                     nearest[0]
2092                 ) )
2093                 zones_overlapping += 1
2094         else:
2095             qalog.append("%s: no centroid\n" % zone)
2096             zones_nocentroid += 1
2097         if not zones.has_option(zone, "description"):
2098             qalog.append("%s: no description\n" % zone)
2099             zones_nodescription += 1
2100         if not zones.has_option(zone, "tz") or not zones.get(
2101                 zone, "tz") in zoneinfo.available_timezones():
2102             qalog.append("%s: no time zone\n" % zone)
2103             zones_notz += 1
2104         if not zones.has_option(zone, "zone_forecast"):
2105             qalog.append("%s: no forecast\n" % zone)
2106             zones_noforecast += 1
2107     if os.path.exists(qalog_fn):
2108         os.rename(qalog_fn, "%s_old"%qalog_fn)
2109     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2110     import time
2111     qalog_fd.write(
2112         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
2113         '# use, copy, modify, and distribute this software is granted under terms\n'
2114         '# provided in the LICENSE file distributed with this software.\n\n'
2115         % time.gmtime().tm_year)
2116     qalog_fd.writelines(qalog)
2117     qalog_fd.close()
2118     if qalog:
2119         print("issues found (see %s for details):"%qalog_fn)
2120         if airports_badstation:
2121             print("   %s airports with invalid station"%airports_badstation)
2122         if airports_nostation:
2123             print("   %s airports with no station"%airports_nostation)
2124         if places_nocentroid:
2125             print("   %s places with no centroid"%places_nocentroid)
2126         if places_nodescription:
2127             print("   %s places with no description"%places_nodescription)
2128         if stations_nodescription:
2129             print("   %s stations with no description"%stations_nodescription)
2130         if stations_nolocation:
2131             print("   %s stations with no location"%stations_nolocation)
2132         if stations_nometar:
2133             print("   %s stations with no METAR"%stations_nometar)
2134         if zctas_nocentroid:
2135             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2136         if zones_nocentroid:
2137             print("   %s zones with no centroid"%zones_nocentroid)
2138         if zones_nodescription:
2139             print("   %s zones with no description"%zones_nodescription)
2140         if zones_notz:
2141             print("   %s zones with no time zone"%zones_notz)
2142         if zones_noforecast:
2143             print("   %s zones with no forecast"%zones_noforecast)
2144         if zones_overlapping:
2145             print("   %s zones within one km of another"%zones_overlapping)
2146     else: print("no issues found.")
2147     print("Indexing complete!")