weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2012 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.0"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 def pyversion(ref=None):
  15     """Determine the Python version and optionally compare to a reference."""
  16     import platform
  17     ver = platform.python_version()
  18     if ref:
  19         return [
  20             int(x) for x in ver.split(".")[:2]
  21         ] >= [
  22             int(x) for x in ref.split(".")[:2]
  23         ]
  24     else: return ver
  25
  26 class Selections:
  27     """An object to contain selection data."""
  28     def __init__(self):
  29         """Store the config, options and arguments."""
  30         self.config = get_config()
  31         self.options, self.arguments = get_options(self.config)
  32         if self.get_bool("cache") and self.get_bool("cache_search") \
  33             and not self.get_bool("longlist"):
  34             integrate_search_cache(
  35                 self.config,
  36                 self.get("cachedir"),
  37                 self.get("setpath")
  38             )
  39         if not self.arguments:
  40             if "id" in self.options.__dict__ \
  41                 and self.options.__dict__["id"]:
  42                 self.arguments.append( self.options.__dict__["id"] )
  43                 del( self.options.__dict__["id"] )
  44                 import sys
  45                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  46                 sys.stderr.write(message)
  47             elif "city" in self.options.__dict__ \
  48                 and self.options.__dict__["city"] \
  49                 and "st" in self.options.__dict__ \
  50                 and self.options.__dict__["st"]:
  51                 self.arguments.append(
  52                     "^%s city, %s" % (
  53                         self.options.__dict__["city"],
  54                         self.options.__dict__["st"]
  55                     )
  56                 )
  57                 del( self.options.__dict__["city"] )
  58                 del( self.options.__dict__["st"] )
  59                 import sys
  60                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  61                 sys.stderr.write(message)
  62     def get(self, option, argument=None):
  63         """Retrieve data from the config or options."""
  64         if argument:
  65             if self.config.has_section(argument) and (
  66                 self.config.has_option(argument, "city") \
  67                     or self.config.has_option(argument, "id") \
  68                     or self.config.has_option(argument, "st")
  69             ):
  70                 self.config.remove_section(argument)
  71                 import sys
  72                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  73                 sys.stderr.write(message)
  74             if not self.config.has_section(argument):
  75                 guessed = guess(
  76                     argument,
  77                     path=self.get("setpath"),
  78                     info=self.get("info"),
  79                     cache_search=(
  80                         self.get("cache") and self.get("cache_search")
  81                     ),
  82                     cachedir=self.get("cachedir")
  83                 )
  84                 self.config.add_section(argument)
  85                 for item in guessed.items():
  86                     self.config.set(argument, *item)
  87             if self.config.has_option(argument, option):
  88                 return self.config.get(argument, option)
  89         if option in self.options.__dict__:
  90             return self.options.__dict__[option]
  91         else:
  92             import os, sys
  93             message = "%s error: no URI defined for %s\n" % (
  94                 os.path.basename( sys.argv[0] ),
  95                 option
  96             )
  97             sys.stderr.write(message)
  98             exit(1)
  99     def get_bool(self, option, argument=None):
 100         """Get data and coerce to a boolean if necessary."""
 101         return bool(self.get(option, argument))
 102     def getint(self, option, argument=None):
 103         """Get data and coerce to an integer if necessary."""
 104         value = self.get(option, argument)
 105         if value: return int(value)
 106         else: return 0
 107
 108 def average(coords):
 109     """Average a list of coordinates."""
 110     x = 0
 111     y = 0
 112     for coord in coords:
 113         x += coord[0]
 114         y += coord[1]
 115     count = len(coords)
 116     return (x/count, y/count)
 117
 118 def filter_units(line, units="imperial"):
 119     """Filter or convert units in a line of text between US/UK and metric."""
 120     import re
 121     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 122     # "X in. Hg (Y hPa)"
 123     dual_p = re.match(
 124         "(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 125         line
 126     )
 127     if dual_p:
 128         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 129         if units == "imperial": line = preamble + in_hg + trailer
 130         elif units == "metric": line = preamble + hpa + trailer
 131     # filter lines with both temperatures in the form of "X F (Y C)"
 132     dual_t = re.match(
 133         "(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 134         line
 135     )
 136     if dual_t:
 137         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 138         if units == "imperial": line = preamble + fahrenheit + trailer
 139         elif units == "metric": line = preamble + celsius + trailer
 140     # if metric is desired, convert distances in the form of "X mile(s)" to
 141     # "Y kilometer(s)"
 142     if units == "metric":
 143         imperial_d = re.match(
 144             "(.* )(\d+)( mile\(s\))(.*)",
 145             line
 146         )
 147         if imperial_d:
 148             preamble, mi, m_u, trailer = imperial_d.groups()
 149             line = preamble + str(int(round(int(mi)*1.609344))) \
 150                 + " kilometer(s)" + trailer
 151     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 152     # desired, convert to "Z KPH"
 153     imperial_s = re.match(
 154         "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 155         line
 156     )
 157     if imperial_s:
 158         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 159         if units == "imperial": line = preamble + mph + m_u + trailer
 160         elif units == "metric":
 161             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 162                 trailer
 163     imperial_s = re.match(
 164         "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 165         line
 166     )
 167     if imperial_s:
 168         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 169         if units == "imperial": line = preamble + mph + m_u + trailer
 170         elif units == "metric":
 171             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 172                 trailer
 173     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 174     # metric is desired, convert to "Y C"
 175     imperial_t = re.match(
 176         "(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 177         line
 178     )
 179     if imperial_t:
 180         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 181         if units == "imperial":
 182             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 183         elif units == "metric":
 184             line = preamble + parameter \
 185                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 186                 + trailer
 187     # hand off the resulting line
 188     return line
 189
 190 def get_uri(
 191     uri,
 192     ignore_fail=False,
 193     cache_data=False,
 194     cacheage=900,
 195     cachedir="."
 196 ):
 197     """Return a string containing the results of a URI GET."""
 198     if pyversion("3"):
 199         import urllib, urllib.error, urllib.request
 200         URLError = urllib.error.URLError
 201         urlopen = urllib.request.urlopen
 202     else:
 203         import urllib2 as urllib
 204         URLError = urllib.URLError
 205         urlopen = urllib.urlopen
 206     import os, time
 207     if cache_data:
 208         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 209         if not os.path.exists(dcachedir):
 210             try: os.makedirs(dcachedir)
 211             except (IOError, OSError): pass
 212         dcache_fn = os.path.join(
 213             dcachedir,
 214             uri.split(":")[1].replace("/","_")
 215         )
 216     now = time.time()
 217     if cache_data and os.access(dcache_fn, os.R_OK) \
 218         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 219         dcache_fd = open(dcache_fn)
 220         data = dcache_fd.read()
 221         dcache_fd.close()
 222     else:
 223         try:
 224             if pyversion("3"): data = urlopen(uri).read().decode("utf-8")
 225             else: data = urlopen(uri).read()
 226         except URLError:
 227             if ignore_fail: return ""
 228             else:
 229                 import os, sys, traceback
 230                 message = "%s error: failed to retrieve\n   %s\n   %s" % (
 231                         os.path.basename( sys.argv[0] ),
 232                         uri,
 233                         traceback.format_exception_only(
 234                             sys.exc_type,
 235                             sys.exc_value
 236                         )[0]
 237                     )
 238                 sys.stderr.write(message)
 239                 sys.exit(1)
 240         if cache_data:
 241             try:
 242                 import codecs
 243                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 244                 dcache_fd.write(data)
 245                 dcache_fd.close()
 246             except (IOError, OSError): pass
 247     return data
 248
 249 def get_metar(
 250     uri=None,
 251     verbose=False,
 252     quiet=False,
 253     headers=None,
 254     imperial=False,
 255     metric=False,
 256     cache_data=False,
 257     cacheage=900,
 258     cachedir="."
 259 ):
 260     """Return a summarized METAR for the specified station."""
 261     if not uri:
 262         import os, sys
 263         message = "%s error: METAR URI required for conditions\n" % \
 264             os.path.basename( sys.argv[0] )
 265         sys.stderr.write(message)
 266         sys.exit(1)
 267     metar = get_uri(
 268         uri,
 269         cache_data=cache_data,
 270         cacheage=cacheage,
 271         cachedir=cachedir
 272     )
 273     if pyversion("3") and type(metar) is bytes: metar = metar.decode("utf-8")
 274     if verbose: return metar
 275     else:
 276         import re
 277         lines = metar.split("\n")
 278         if not headers:
 279             headers = \
 280                 "relative_humidity," \
 281                 + "precipitation_last_hour," \
 282                 + "sky conditions," \
 283                 + "temperature," \
 284                 + "heat index," \
 285                 + "windchill," \
 286                 + "weather," \
 287                 + "wind"
 288         headerlist = headers.lower().replace("_"," ").split(",")
 289         output = []
 290         if not quiet:
 291             title = "Current conditions at %s"
 292             place = lines[0].split(", ")
 293             if len(place) > 1:
 294                 place = "%s, %s" % ( place[0].title(), place[1] )
 295             else: place = "<UNKNOWN>"
 296             output.append(title%place)
 297             output.append("Last updated " + lines[1])
 298         header_match = False
 299         for header in headerlist:
 300             for line in lines:
 301                 if line.lower().startswith(header + ":"):
 302                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 303                     if imperial: line = filter_units(line, units="imperial")
 304                     elif metric: line = filter_units(line, units="metric")
 305                     if quiet: output.append(line)
 306                     else: output.append("   " + line)
 307                     header_match = True
 308         if not header_match:
 309             output.append(
 310                 "(no conditions matched your header list, try with --verbose)"
 311             )
 312         return "\n".join(output)
 313
 314 def get_alert(
 315     uri=None,
 316     verbose=False,
 317     quiet=False,
 318     cache_data=False,
 319     cacheage=900,
 320     cachedir="."
 321 ):
 322     """Return alert notice for the specified URI."""
 323     if not uri:
 324         import os, sys
 325         message = "%s error: Alert URI required for alerts\n" % \
 326             os.path.basename( sys.argv[0] )
 327         sys.stderr.write(message)
 328         sys.exit(1)
 329     alert = get_uri(
 330         uri,
 331         ignore_fail=True,
 332         cache_data=cache_data,
 333         cacheage=cacheage,
 334         cachedir=cachedir
 335     ).strip()
 336     if pyversion("3") and type(alert) is bytes: alert = alert.decode("utf-8")
 337     if alert:
 338         if verbose: return alert
 339         else:
 340             if alert.find("\nNATIONAL WEATHER SERVICE") == -1:
 341                 muted = False
 342             else:
 343                 muted = True
 344             lines = alert.split("\n")
 345             import time
 346             valid_time = time.strftime("%Y%m%d%H%M")
 347             output = []
 348             for line in lines:
 349                 if line.startswith("Expires:") \
 350                     and "Expires:" + valid_time > line:
 351                     return ""
 352                 if muted and line.startswith("NATIONAL WEATHER SERVICE"):
 353                     muted = False
 354                     line = ""
 355                 elif line == "&&":
 356                     line = ""
 357                 elif line == "$$":
 358                     muted = True
 359                 if line and not muted:
 360                     if quiet: output.append(line)
 361                     else: output.append("   " + line)
 362             return "\n".join(output)
 363
 364 def get_options(config):
 365     """Parse the options passed on the command line."""
 366
 367     # for optparse's builtin -h/--help option
 368     usage = \
 369         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 370
 371     # for optparse's builtin --version option
 372     verstring = "%prog " + weather_version
 373
 374     # create the parser
 375     import optparse
 376     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 377     # separate options object from list of arguments and return both
 378
 379     # the -a/--alert option
 380     if config.has_option("default", "alert"):
 381         default_alert = bool(config.get("default", "alert"))
 382     else: default_alert = False
 383     option_parser.add_option("-a", "--alert",
 384         dest="alert",
 385         action="store_true",
 386         default=default_alert,
 387         help="include local alert notices")
 388
 389     # the --atypes option
 390     if config.has_option("default", "atypes"):
 391         default_atypes = config.get("default", "atypes")
 392     else:
 393         default_atypes = \
 394             "coastal_flood_statement," \
 395             + "flash_flood_statement," \
 396             + "flash_flood_warning," \
 397             + "flash_flood_watch," \
 398             + "flood_statement," \
 399             + "flood_warning," \
 400             + "marine_weather_statement," \
 401             + "river_statement," \
 402             + "severe_thunderstorm_warning," \
 403             + "severe_weather_statement," \
 404             + "short_term_forecast," \
 405             + "special_marine_warning," \
 406             + "special_weather_statement," \
 407             + "tornado_warning," \
 408             + "urgent_weather_message"
 409     option_parser.add_option("--atypes",
 410         dest="atypes",
 411         default=default_atypes,
 412         help="list of alert notification types to display")
 413
 414     # the --build-sets option
 415     option_parser.add_option("--build-sets",
 416         dest="build_sets",
 417         action="store_true",
 418         default=False,
 419         help="(re)build location correlation sets")
 420
 421     # the --cacheage option
 422     if config.has_option("default", "cacheage"):
 423         default_cacheage = config.getint("default", "cacheage")
 424     else: default_cacheage = 900
 425     option_parser.add_option("--cacheage",
 426         dest="cacheage",
 427         default=default_cacheage,
 428         help="duration in seconds to refresh cached data")
 429
 430     # the --cachedir option
 431     if config.has_option("default", "cachedir"):
 432         default_cachedir = config.get("default", "cachedir")
 433     else: default_cachedir = "~/.weather"
 434     option_parser.add_option("--cachedir",
 435         dest="cachedir",
 436         default=default_cachedir,
 437         help="directory for storing cached searches and data")
 438
 439     # the -f/--forecast option
 440     if config.has_option("default", "forecast"):
 441         default_forecast = bool(config.get("default", "forecast"))
 442     else: default_forecast = False
 443     option_parser.add_option("-f", "--forecast",
 444         dest="forecast",
 445         action="store_true",
 446         default=default_forecast,
 447         help="include a local forecast")
 448
 449     # the --headers option
 450     if config.has_option("default", "headers"):
 451         default_headers = config.get("default", "headers")
 452     else:
 453         default_headers = \
 454             "temperature," \
 455             + "relative_humidity," \
 456             + "wind," \
 457             + "heat_index," \
 458             + "windchill," \
 459             + "weather," \
 460             + "sky_conditions," \
 461             + "precipitation_last_hour"
 462     option_parser.add_option("--headers",
 463         dest="headers",
 464         default=default_headers,
 465         help="list of conditions headers to display")
 466
 467     # the --imperial option
 468     if config.has_option("default", "imperial"):
 469         default_imperial = bool(config.get("default", "imperial"))
 470     else: default_imperial = False
 471     option_parser.add_option("--imperial",
 472         dest="imperial",
 473         action="store_true",
 474         default=default_imperial,
 475         help="filter/convert conditions for US/UK units")
 476
 477     # the --info option
 478     option_parser.add_option("--info",
 479         dest="info",
 480         action="store_true",
 481         default=False,
 482         help="output detailed information for your search")
 483
 484     # the -l/--list option
 485     option_parser.add_option("-l", "--list",
 486         dest="list",
 487         action="store_true",
 488         default=False,
 489         help="list all configured aliases and cached searches")
 490
 491     # the --longlist option
 492     option_parser.add_option("--longlist",
 493         dest="longlist",
 494         action="store_true",
 495         default=False,
 496         help="display details of all configured aliases")
 497
 498     # the -m/--metric option
 499     if config.has_option("default", "metric"):
 500         default_metric = bool(config.get("default", "metric"))
 501     else: default_metric = False
 502     option_parser.add_option("-m", "--metric",
 503         dest="metric",
 504         action="store_true",
 505         default=default_metric,
 506         help="filter/convert conditions for metric units")
 507
 508     # the -n/--no-conditions option
 509     if config.has_option("default", "conditions"):
 510         default_conditions = bool(config.get("default", "conditions"))
 511     else: default_conditions = True
 512     option_parser.add_option("-n", "--no-conditions",
 513         dest="conditions",
 514         action="store_false",
 515         default=default_conditions,
 516         help="disable output of current conditions")
 517
 518     # the --no-cache option
 519     if config.has_option("default", "cache"):
 520         default_cache = bool(config.get("default", "cache"))
 521     else: default_cache = True
 522     option_parser.add_option("--no-cache",
 523         dest="cache",
 524         action="store_false",
 525         default=True,
 526         help="disable all caching (searches and data)")
 527
 528     # the --no-cache-data option
 529     if config.has_option("default", "cache_data"):
 530         default_cache_data = bool(config.get("default", "cache_data"))
 531     else: default_cache_data = True
 532     option_parser.add_option("--no-cache-data",
 533         dest="cache_data",
 534         action="store_false",
 535         default=True,
 536         help="disable retrieved data caching")
 537
 538     # the --no-cache-search option
 539     if config.has_option("default", "cache_search"):
 540         default_cache_search = bool(config.get("default", "cache_search"))
 541     else: default_cache_search = True
 542     option_parser.add_option("--no-cache-search",
 543         dest="cache_search",
 544         action="store_false",
 545         default=True,
 546         help="disable search result caching")
 547
 548     # the -q/--quiet option
 549     if config.has_option("default", "quiet"):
 550         default_quiet = bool(config.get("default", "quiet"))
 551     else: default_quiet = False
 552     option_parser.add_option("-q", "--quiet",
 553         dest="quiet",
 554         action="store_true",
 555         default=default_quiet,
 556         help="skip preambles and don't indent")
 557
 558     # the --setpath option
 559     if config.has_option("default", "setpath"):
 560         default_setpath = config.get("default", "setpath")
 561     else: default_setpath = ".:~/.weather"
 562     option_parser.add_option("--setpath",
 563         dest="setpath",
 564         default=default_setpath,
 565         help="directory search path for correlation sets")
 566
 567     # the -v/--verbose option
 568     if config.has_option("default", "verbose"):
 569         default_verbose = bool(config.get("default", "verbose"))
 570     else: default_verbose = False
 571     option_parser.add_option("-v", "--verbose",
 572         dest="verbose",
 573         action="store_true",
 574         default=default_verbose,
 575         help="show full decoded feeds")
 576
 577     # deprecated options
 578     if config.has_option("default", "city"):
 579         default_city = config.get("default", "city")
 580     else: default_city = ""
 581     option_parser.add_option("-c", "--city",
 582         dest="city",
 583         default=default_city,
 584         help=optparse.SUPPRESS_HELP)
 585     if config.has_option("default", "id"):
 586         default_id = config.get("default", "id")
 587     else: default_id = ""
 588     option_parser.add_option("-i", "--id",
 589         dest="id",
 590         default=default_id,
 591         help=optparse.SUPPRESS_HELP)
 592     if config.has_option("default", "st"):
 593         default_st = config.get("default", "st")
 594     else: default_st = ""
 595     option_parser.add_option("-s", "--st",
 596         dest="st",
 597         default=default_st,
 598         help=optparse.SUPPRESS_HELP)
 599
 600     options, arguments = option_parser.parse_args()
 601     return options, arguments
 602
 603 def get_config():
 604     """Parse the aliases and configuration."""
 605     if pyversion("3"): import configparser
 606     else: import ConfigParser as configparser
 607     config = configparser.ConfigParser()
 608     import os
 609     rcfiles = [
 610         "/etc/weatherrc",
 611         os.path.expanduser("~/.weather/weatherrc"),
 612         os.path.expanduser("~/.weatherrc"),
 613         "weatherrc"
 614         ]
 615     for rcfile in rcfiles:
 616         if os.access(rcfile, os.R_OK): config.read(rcfile)
 617     for section in config.sections():
 618         if section != section.lower():
 619             if config.has_section(section.lower()):
 620                 config.remove_section(section.lower())
 621             config.add_section(section.lower())
 622             for option,value in config.items(section):
 623                 config.set(section.lower(), option, value)
 624     return config
 625
 626 def integrate_search_cache(config, cachedir, setpath):
 627     """Add cached search results into the configuration."""
 628     if pyversion("3"): import configparser
 629     else: import ConfigParser as configparser
 630     import os, time
 631     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 632     if not os.access(scache_fn, os.R_OK): return config
 633     scache_fd = open(scache_fn)
 634     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 635     scache_fd.close()
 636     now = time.time()
 637     datafiles = data_index(setpath)
 638     if datafiles:
 639         data_freshness = sorted(
 640             [ x[1] for x in datafiles.values() ],
 641             reverse=True
 642         )[0]
 643     else: data_freshness = now
 644     if created < data_freshness <= now:
 645         try:
 646             os.remove(scache_fn)
 647             print( "[clearing outdated %s]" % scache_fn )
 648         except (IOError, OSError):
 649             pass
 650         return config
 651     scache = configparser.ConfigParser()
 652     scache.read(scache_fn)
 653     for section in scache.sections():
 654         if not config.has_section(section):
 655             config.add_section(section)
 656             for option,value in scache.items(section):
 657                 config.set(section, option, value)
 658     return config
 659
 660 def list_aliases(config, detail=False):
 661     """Return a formatted list of aliases defined in the config."""
 662     if detail:
 663         output = "\n# configured alias details..."
 664         for section in sorted(config.sections()):
 665             output += "\n\n[%s]" % section
 666             for item in sorted(config.items(section)):
 667                 output += "\n%s = %s" % item
 668         output += "\n"
 669     else:
 670         output = "configured aliases and cached searches..."
 671         for section in sorted(config.sections()):
 672             if config.has_option(section, "description"):
 673                 description = config.get(section, "description")
 674             else: description = "(no description provided)"
 675             output += "\n   %s: %s" % (section, description)
 676     return output
 677
 678 def data_index(path):
 679     import os
 680     datafiles = {}
 681     for filename in ("airports", "places", "stations", "zctas", "zones"):
 682         for dirname in path.split(":"):
 683             for extension in ("", ".gz", ".txt"):
 684                 candidate = os.path.expanduser(
 685                     os.path.join( dirname, "".join( (filename, extension) ) )
 686                 )
 687                 if os.path.exists(candidate):
 688                     datafiles[filename] = (
 689                         candidate,
 690                         os.stat(candidate).st_mtime
 691                     )
 692                     break
 693     return datafiles
 694
 695 def guess(
 696     expression,
 697     path=".",
 698     max_results=20,
 699     info=False,
 700     cache_search=False,
 701     cacheage=900,
 702     cachedir="."
 703 ):
 704     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 705     import codecs, datetime, time, os, re, sys
 706     if pyversion("3"): import configparser
 707     else: import ConfigParser as configparser
 708     datafiles = data_index(path)
 709     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 710     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 711     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 712     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 713     elif re.match(
 714         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 715         expression
 716     ):
 717         searchtype = "coordinates"
 718     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 719     else:
 720         searchtype = "name"
 721         cache_search = False
 722     if cache_search: action = "caching"
 723     else: action = "using"
 724     if info:
 725         scores = [
 726             (0.005, "bad"),
 727             (0.025, "poor"),
 728             (0.160, "suspect"),
 729             (0.500, "mediocre"),
 730             (0.840, "good"),
 731             (0.975, "great"),
 732             (0.995, "excellent"),
 733             (1.000, "ideal"),
 734         ]
 735     print("Searching via %s..."%searchtype)
 736     stations = configparser.ConfigParser()
 737     dataname = "stations"
 738     if dataname in datafiles:
 739         datafile = datafiles[dataname][0]
 740         if datafile.endswith(".gz"):
 741             import gzip
 742             stations.readfp( gzip.open(datafile) )
 743         else:
 744             stations.read(datafile)
 745     else:
 746         message = "%s error: can't find \"%s\" data file\n" % (
 747             os.path.basename( sys.argv[0] ),
 748             dataname
 749         )
 750         sys.stderr.write(message)
 751         exit(1)
 752     zones = configparser.ConfigParser()
 753     dataname = "zones"
 754     if dataname in datafiles:
 755         datafile = datafiles[dataname][0]
 756         if datafile.endswith(".gz"):
 757             import gzip
 758             zones.readfp( gzip.open(datafile) )
 759         else:
 760             zones.read(datafile)
 761     else:
 762         message = "%s error: can't find \"%s\" data file\n" % (
 763             os.path.basename( sys.argv[0] ),
 764             dataname
 765         )
 766         sys.stderr.write(message)
 767         exit(1)
 768     search = None
 769     station = ("", 0)
 770     zone = ("", 0)
 771     dataset = None
 772     possibilities = []
 773     uris = {}
 774     if searchtype == "airport":
 775         expression = expression.lower()
 776         airports = configparser.ConfigParser()
 777         dataname = "airports"
 778         if dataname in datafiles:
 779             datafile = datafiles[dataname][0]
 780             if datafile.endswith(".gz"):
 781                 import gzip
 782                 airports.readfp( gzip.open(datafile) )
 783             else:
 784                 airports.read(datafile)
 785         else:
 786             message = "%s error: can't find \"%s\" data file\n" % (
 787                 os.path.basename( sys.argv[0] ),
 788                 dataname
 789             )
 790             sys.stderr.write(message)
 791             exit(1)
 792         if airports.has_section(expression) \
 793             and airports.has_option(expression, "station"):
 794             search = (expression, "IATA/FAA airport code %s" % expression)
 795             station = ( airports.get(expression, "station"), 0 )
 796             if stations.has_option(station[0], "zone"):
 797                 zone = eval( stations.get(station[0], "zone") )
 798                 dataset = stations
 799             if not info and stations.has_option( station[0], "description" ):
 800                 print(
 801                     "[%s result %s]" % (
 802                         action,
 803                         stations.get(station[0], "description")
 804                     )
 805                 )
 806         else:
 807             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 808                 expression,
 809                 datafiles["airports"][0]
 810             )
 811             sys.stderr.write(message)
 812             exit(1)
 813     elif searchtype == "station":
 814         expression = expression.lower()
 815         if stations.has_section(expression):
 816             station = (expression, 0)
 817             if not search:
 818                 search = (expression, "ICAO station code %s" % expression)
 819             if stations.has_option(expression, "zone"):
 820                 zone = eval( stations.get(expression, "zone") )
 821                 dataset = stations
 822             if not info and stations.has_option(expression, "description"):
 823                 print(
 824                     "[%s result %s]" % (
 825                         action,
 826                         stations.get(expression, "description")
 827                     )
 828                 )
 829         else:
 830             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 831                 expression,
 832                 datafiles["stations"][0]
 833             )
 834             sys.stderr.write(message)
 835             exit(1)
 836     elif searchtype == "zone":
 837         expression = expression.lower()
 838         if zones.has_section(expression) \
 839             and zones.has_option(expression, "station"):
 840             zone = (expression, 0)
 841             station = eval( zones.get(expression, "station") )
 842             dataset = zones
 843             search = (expression, "NWS/NOAA weather zone %s" % expression)
 844             if not info and zones.has_option(expression, "description"):
 845                 print(
 846                     "[%s result %s]" % (
 847                         action,
 848                         zones.get(expression, "description")
 849                     )
 850                 )
 851         else:
 852             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 853                 expression,
 854                 datafiles["zones"][0]
 855             )
 856             sys.stderr.write(message)
 857             exit(1)
 858     elif searchtype == "ZCTA":
 859         zctas = configparser.ConfigParser()
 860         dataname = "zctas"
 861         if dataname in datafiles:
 862             datafile = datafiles[dataname][0]
 863             if datafile.endswith(".gz"):
 864                 import gzip
 865                 zctas.readfp( gzip.open(datafile) )
 866             else:
 867                 zctas.read(datafile)
 868         else:
 869             message = "%s error: can't find \"%s\" data file\n" % (
 870                 os.path.basename( sys.argv[0] ),
 871                 dataname
 872             )
 873             sys.stderr.write(message)
 874             exit(1)
 875         dataset = zctas
 876         if zctas.has_section(expression) \
 877             and zctas.has_option(expression, "station"):
 878             station = eval( zctas.get(expression, "station") )
 879             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 880             if zctas.has_option(expression, "zone"):
 881                 zone = eval( zctas.get(expression, "zone") )
 882         else:
 883             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 884                 expression,
 885                 datafiles["zctas"][0]
 886             )
 887             sys.stderr.write(message)
 888             exit(1)
 889     elif searchtype == "coordinates":
 890         search = (expression, "Geographic coordinates %s" % expression)
 891         stationtable = {}
 892         for station in stations.sections():
 893             if stations.has_option(station, "location"):
 894                 stationtable[station] = {
 895                     "location": eval( stations.get(station, "location") )
 896                 }
 897         station = closest( gecos(expression), stationtable, "location", 0.1 )
 898         if not station[0]:
 899             message = "No ICAO weather station found near %s.\n" % expression
 900             sys.stderr.write(message)
 901             exit(1)
 902         zonetable = {}
 903         for zone in zones.sections():
 904             if zones.has_option(zone, "centroid"):
 905                 zonetable[zone] = {
 906                     "centroid": eval( zones.get(zone, "centroid") )
 907                 }
 908         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 909         if not zone[0]:
 910             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 911                 % expression
 912             sys.stderr.write(message)
 913     elif searchtype in ("FIPS", "name"):
 914         places = configparser.ConfigParser()
 915         dataname = "places"
 916         if dataname in datafiles:
 917             datafile = datafiles[dataname][0]
 918             if datafile.endswith(".gz"):
 919                 import gzip
 920                 places.readfp( gzip.open(datafile) )
 921             else:
 922                 places.read(datafile)
 923         else:
 924             message = "%s error: can't find \"%s\" data file\n" % (
 925                 os.path.basename( sys.argv[0] ),
 926                 dataname
 927             )
 928             sys.stderr.write(message)
 929             exit(1)
 930         dataset = places
 931         place = expression.lower()
 932         if places.has_section(place) and places.has_option(place, "station"):
 933             station = eval( places.get(place, "station") )
 934             search = (expression, "Census Place %s" % expression)
 935             if places.has_option(place, "description"):
 936                 search = (
 937                     search[0],
 938                     search[1] + ", %s" % places.get(place, "description")
 939                 )
 940             if places.has_option(place, "zone"):
 941                 zone = eval( places.get(place, "zone") )
 942             if not info and places.has_option(place, "description"):
 943                 print(
 944                     "[%s result %s]" % (
 945                         action,
 946                         places.get(place, "description")
 947                     )
 948                 )
 949         else:
 950             for place in places.sections():
 951                 if places.has_option(place, "description") \
 952                     and places.has_option(place, "station") \
 953                     and re.search(
 954                         expression,
 955                         places.get(place, "description"),
 956                         re.I
 957                     ):
 958                         possibilities.append(place)
 959             for place in stations.sections():
 960                 if stations.has_option(place, "description") \
 961                     and re.search(
 962                         expression,
 963                         stations.get(place, "description"),
 964                         re.I
 965                     ):
 966                         possibilities.append(place)
 967             for place in zones.sections():
 968                 if zones.has_option(place, "description") \
 969                     and zones.has_option(place, "station") \
 970                     and re.search(
 971                         expression,
 972                         zones.get(place, "description"),
 973                         re.I
 974                     ):
 975                         possibilities.append(place)
 976             if len(possibilities) == 1:
 977                 place = possibilities[0]
 978                 if places.has_section(place):
 979                     station = eval( places.get(place, "station") )
 980                     description = places.get(place, "description")
 981                     if places.has_option(place, "zone"):
 982                         zone = eval( places.get(place, "zone" ) )
 983                     search = ( expression, "%s: %s" % (place, description) )
 984                 elif stations.has_section(place):
 985                     station = (place, 0.0)
 986                     description = stations.get(place, "description")
 987                     if stations.has_option(place, "zone"):
 988                         zone = eval( stations.get(place, "zone" ) )
 989                     search = ( expression, "ICAO station code %s" % place )
 990                 elif zones.has_section(place):
 991                     station = eval( zones.get(place, "station") )
 992                     description = zones.get(place, "description")
 993                     zone = (place, 0.0)
 994                     search = ( expression, "NWS/NOAA weather zone %s" % place )
 995                 if not info: print( "[%s result %s]" % (action, description) )
 996             if not possibilities and not station[0]:
 997                 message = "No FIPS code/census area match in the %s file.\n" % (
 998                     datafiles["places"][0]
 999                 )
1000                 sys.stderr.write(message)
1001                 exit(1)
1002     if station[0]:
1003         uris["metar"] = stations.get( station[0], "metar" )
1004         if zone[0]:
1005             for key,value in zones.items( zone[0] ):
1006                 if key not in ("centroid", "description", "station"):
1007                     uris[key] = value
1008     elif possibilities:
1009         count = len(possibilities)
1010         if count <= max_results:
1011             print( "Your search is ambiguous, returning %s matches:" % count )
1012             for place in sorted(possibilities):
1013                 if places.has_section(place):
1014                     print(
1015                         "   [%s] %s" % (
1016                             place,
1017                             places.get(place, "description")
1018                         )
1019                     )
1020                 elif stations.has_section(place):
1021                     print(
1022                         "   [%s] %s" % (
1023                             place,
1024                             stations.get(place, "description")
1025                         )
1026                     )
1027                 elif zones.has_section(place):
1028                     print(
1029                         "   [%s] %s" % (
1030                             place,
1031                             zones.get(place, "description")
1032                         )
1033                     )
1034         else:
1035             print(
1036                 "Your search is too ambiguous, returning %s matches." % count
1037             )
1038         exit(0)
1039     if info:
1040         stationlist = []
1041         zonelist = []
1042         if dataset:
1043             for section in dataset.sections():
1044                 if dataset.has_option(section, "station"):
1045                     stationlist.append(
1046                         eval( dataset.get(section, "station") )[1]
1047                     )
1048                 if dataset.has_option(section, "zone"):
1049                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1050         stationlist.sort()
1051         zonelist.sort()
1052         scount = len(stationlist)
1053         zcount = len(zonelist)
1054         sranks = []
1055         zranks = []
1056         for score in scores:
1057             if stationlist:
1058                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1059             if zonelist:
1060                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1061         description = search[1]
1062         uris["description"] = description
1063         print(
1064             "%s\n%s" % ( description, "-" * len(description) )
1065         )
1066         print(
1067             "%s: %s" % (
1068                 station[0],
1069                 stations.get( station[0], "description" )
1070             )
1071         )
1072         km = radian_to_km*station[1]
1073         mi = radian_to_mi*station[1]
1074         if sranks and not description.startswith("ICAO station code "):
1075             for index in range(0, len(scores)):
1076                 if station[1] >= sranks[index]:
1077                     score = scores[index][1]
1078                     break
1079             print(
1080                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1081             )
1082         elif searchtype is "coordinates":
1083             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1084         if zone[0]:
1085             print(
1086                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1087             )
1088         km = radian_to_km*zone[1]
1089         mi = radian_to_mi*zone[1]
1090         if zranks and not description.startswith("NWS/NOAA weather zone "):
1091             for index in range(0, len(scores)):
1092                 if zone[1] >= zranks[index]:
1093                     score = scores[index][1]
1094                     break
1095             print(
1096                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1097             )
1098         elif searchtype is "coordinates" and zone[0]:
1099             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1100     if cache_search:
1101         now = time.time()
1102         nowstamp = "%s (%s)" % (
1103             now,
1104             datetime.datetime.isoformat(
1105                 datetime.datetime.fromtimestamp(now),
1106                 " "
1107             )
1108         )
1109         search_cache = ["\n"]
1110         search_cache.append( "[%s]\n" % search[0] )
1111         search_cache.append( "description = cached %s\n" % nowstamp )
1112         for uriname in sorted(uris.keys()):
1113             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1114         real_cachedir = os.path.expanduser(cachedir)
1115         if not os.path.exists(real_cachedir):
1116             try: os.makedirs(real_cachedir)
1117             except (IOError, OSError): pass
1118         scache_fn = os.path.join(real_cachedir, "searches")
1119         if not os.path.exists(scache_fn):
1120             then = sorted(
1121                     [ x[1] for x in datafiles.values() ],
1122                     reverse=True
1123                 )[0]
1124             thenstamp = "%s (%s)" % (
1125                 then,
1126                 datetime.datetime.isoformat(
1127                     datetime.datetime.fromtimestamp(then),
1128                     " "
1129                 )
1130             )
1131             search_cache.insert(
1132                 0,
1133                 "# based on data files from: %s\n" % thenstamp
1134             )
1135         try:
1136             scache_existing = configparser.ConfigParser()
1137             scache_existing.read(scache_fn)
1138             if not scache_existing.has_section(search[0]):
1139                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1140                 scache_fd.writelines(search_cache)
1141                 scache_fd.close()
1142         except (IOError, OSError): pass
1143     if not info:
1144         return(uris)
1145
1146 def closest(position, nodes, fieldname, angle=None):
1147     import math
1148     if not angle: angle = 2*math.pi
1149     match = None
1150     for name in nodes:
1151         if fieldname in nodes[name]:
1152             node = nodes[name][fieldname]
1153             if node and abs( position[0]-node[0] ) < angle:
1154                 if abs( position[1]-node[1] ) < angle \
1155                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1156                     if position == node:
1157                         angle = 0
1158                         match = name
1159                     else:
1160                         candidate = math.acos(
1161                             math.sin( position[0] ) * math.sin( node[0] ) \
1162                                 + math.cos( position[0] ) \
1163                                 * math.cos( node[0] ) \
1164                                 * math.cos( position[1] - node[1] )
1165                             )
1166                         if candidate < angle:
1167                             angle = candidate
1168                             match = name
1169     if match: match = str(match)
1170     return (match, angle)
1171
1172 def gecos(formatted):
1173     import math, re
1174     coordinates = formatted.split(",")
1175     for coordinate in range(0, 2):
1176         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1177             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1178             coordinates[coordinate].strip().lower()
1179         ).groups()
1180         value = float(degrees)
1181         if minutes: value += float(minutes)/60
1182         if seconds: value += float(seconds)/3600
1183         if hemisphere and hemisphere in "sw": value *= -1
1184         coordinates[coordinate] = math.radians(value)
1185     return tuple(coordinates)
1186
1187 def correlate():
1188     import codecs, datetime, hashlib, os, re, sys, tarfile, time, zipfile
1189     if pyversion("3"): import configparser
1190     else: import ConfigParser as configparser
1191     gcounties_an = "Gaz_counties_national.zip"
1192     gcounties_fn = "Gaz_counties_national.txt"
1193     gcousubs_an = "Gaz_cousubs_national.zip"
1194     gcousubs_fn = "Gaz_cousubs_national.txt"
1195     gplaces_an = "Gaz_places_national.zip"
1196     gplaces_fn = "Gaz_places_national.txt"
1197     gzcta_an = "Gaz_zcta_national.zip"
1198     gzcta_fn = "Gaz_zcta_national.txt"
1199     for filename in os.listdir("."):
1200         if re.match("bp[0-9][0-9][a-z][a-z][0-9][0-9].dbx$", filename):
1201             cpfzcf_fn = filename
1202             break
1203     nsdcccc_fn = "nsd_cccc.txt"
1204     zcatalog_an = "zonecatalog.curr.tar"
1205     metartbl_fn = "metar.tbl"
1206     coopact_fn = "COOP-ACT.TXT"
1207     overrides_fn = "overrides.conf"
1208     overrideslog_fn = "overrides.log"
1209     slist_fn = "slist"
1210     zlist_fn = "zlist"
1211     qalog_fn = "qa.log"
1212     airports_fn = "airports"
1213     places_fn = "places"
1214     stations_fn = "stations"
1215     zctas_fn = "zctas"
1216     zones_fn = "zones"
1217     header = """\
1218 %s
1219 # generated by %s on %s from these public domain sources:
1220 #
1221 # http://www.census.gov/geo/www/gazetteer/gazetteer2010.html
1222 # %s %s %s
1223 # %s %s %s
1224 # %s %s %s
1225 # %s %s %s
1226 #
1227 # http://www.weather.gov/geodata/catalog/wsom/html/cntyzone.htm
1228 # %s %s %s
1229 #
1230 # http://weather.noaa.gov/data/nsd_cccc.txt
1231 # %s %s %s
1232 #
1233 # http://weather.noaa.gov/pub/data/zonecatalog.curr.tar
1234 # %s %s %s
1235 #
1236 # http://www.nco.ncep.noaa.gov/pmb/codes/nwprod/dictionaries/metar.tbl
1237 # %s %s %s
1238 #
1239 # ftp://ftp.ncdc.noaa.gov/pub/data/inventories/COOP-ACT.TXT
1240 # %s %s %s
1241 #
1242 # ...and these manually-generated or hand-compiled adjustments:
1243 # %s %s %s
1244 # %s %s %s
1245 # %s %s %s\
1246 """ % (
1247         weather_copyright,
1248         os.path.basename( sys.argv[0] ),
1249         datetime.date.isoformat(
1250             datetime.datetime.fromtimestamp( time.time() )
1251         ),
1252         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1253         datetime.date.isoformat(
1254             datetime.datetime.fromtimestamp( os.path.getmtime(gcounties_an) )
1255         ),
1256         gcounties_an,
1257         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1258         datetime.date.isoformat(
1259             datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) )
1260         ),
1261         gcousubs_an,
1262         hashlib.md5( open(gplaces_an, "rb").read() ).hexdigest(),
1263         datetime.date.isoformat(
1264             datetime.datetime.fromtimestamp( os.path.getmtime(gplaces_an) )
1265         ),
1266         gplaces_an,
1267         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1268         datetime.date.isoformat(
1269             datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) )
1270         ),
1271         gzcta_an,
1272         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1273         datetime.date.isoformat(
1274             datetime.datetime.fromtimestamp( os.path.getmtime(cpfzcf_fn) )
1275         ),
1276         cpfzcf_fn,
1277         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1278         datetime.date.isoformat(
1279             datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) )
1280         ),
1281         nsdcccc_fn,
1282         hashlib.md5( open(zcatalog_an, "rb").read() ).hexdigest(),
1283         datetime.date.isoformat(
1284             datetime.datetime.fromtimestamp( os.path.getmtime(zcatalog_an) )
1285         ),
1286         zcatalog_an,
1287         hashlib.md5( open(metartbl_fn, "rb").read() ).hexdigest(),
1288         datetime.date.isoformat(
1289             datetime.datetime.fromtimestamp( os.path.getmtime(metartbl_fn) )
1290         ),
1291         metartbl_fn,
1292         hashlib.md5( open(coopact_fn, "rb").read() ).hexdigest(),
1293         datetime.date.isoformat(
1294             datetime.datetime.fromtimestamp( os.path.getmtime(coopact_fn) )
1295         ),
1296         coopact_fn,
1297         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1298         datetime.date.isoformat(
1299             datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) )
1300         ),
1301         overrides_fn,
1302         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1303         datetime.date.isoformat(
1304             datetime.datetime.fromtimestamp( os.path.getmtime(slist_fn) )
1305         ),
1306         slist_fn,
1307         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1308         datetime.date.isoformat(
1309             datetime.datetime.fromtimestamp( os.path.getmtime(zlist_fn) )
1310         ),
1311         zlist_fn
1312     )
1313     airports = {}
1314     places = {}
1315     stations = {}
1316     zctas = {}
1317     zones = {}
1318     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1319     sys.stdout.write(message)
1320     sys.stdout.flush()
1321     count = 0
1322     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU")
1323     for line in gcounties:
1324         fields = line.decode("latin1").strip().split("\t")
1325         if len(fields) == 10 and fields[0] != "STUSPS":
1326             fips = "fips%s" % fields[1]
1327             description = "%s, %s" % ( fields[3], fields[0] )
1328             centroid = gecos( ",".join( fields[8:10] ) )
1329             if fips not in places: places[fips] = {}
1330             places[fips]["centroid"] = centroid
1331             places[fips]["description"] = description
1332             count += 1
1333     gcounties.close()
1334     print("done (%s lines)." % count)
1335     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1336     sys.stdout.write(message)
1337     sys.stdout.flush()
1338     count = 0
1339     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU")
1340     for line in gcousubs:
1341         fields = line.decode("latin1").strip().split("\t")
1342         if len(fields) == 10 and fields[0] != "STUSPS":
1343             fips = "fips%s" % fields[1]
1344             description = "%s, %s" % ( fields[3], fields[0] )
1345             centroid = gecos( ",".join( fields[8:10] ) )
1346             if fips not in places: places[fips] = {}
1347             places[fips]["centroid"] = centroid
1348             places[fips]["description"] = description
1349             count += 1
1350     gcousubs.close()
1351     print("done (%s lines)." % count)
1352     message = "Reading %s:%s..." % (gplaces_an, gplaces_fn)
1353     sys.stdout.write(message)
1354     sys.stdout.flush()
1355     count = 0
1356     gplaces = zipfile.ZipFile(gplaces_an).open(gplaces_fn, "rU")
1357     for line in gplaces:
1358         fields = line.decode("latin1").strip().split("\t")
1359         if len(fields) == 10 and fields[0] != "STUSPS":
1360             fips = "fips%s" % fields[1]
1361             description = "%s, %s" % ( fields[3], fields[0] )
1362             centroid = gecos( ",".join( fields[8:10] ) )
1363             if fips not in places: places[fips] = {}
1364             places[fips]["centroid"] = centroid
1365             places[fips]["description"] = description
1366             count += 1
1367     gplaces.close()
1368     print("done (%s lines)." % count)
1369     message = "Reading %s..." % slist_fn
1370     sys.stdout.write(message)
1371     sys.stdout.flush()
1372     count = 0
1373     slist = codecs.open(slist_fn, "rU")
1374     for line in slist:
1375         icao = line.split("#")[0].strip()
1376         if icao:
1377             stations[icao] = {
1378                 "metar": "http://weather.noaa.gov/pub/data/observations/"\
1379                     + "metar/decoded/%s.TXT" % icao.upper()
1380             }
1381             count += 1
1382     slist.close()
1383     print("done (%s lines)." % count)
1384     message = "Reading %s..." % metartbl_fn
1385     sys.stdout.write(message)
1386     sys.stdout.flush()
1387     count = 0
1388     metartbl = codecs.open(metartbl_fn, "rU")
1389     for line in metartbl:
1390         icao = line[:4].strip().lower()
1391         if icao in stations:
1392             description = []
1393             name = " ".join(
1394                 line[16:48].replace("_", " ").strip().title().split()
1395             )
1396             if name: description.append(name)
1397             st = line[49:51].strip()
1398             if st: description.append(st)
1399             cn = line[52:54].strip()
1400             if cn: description.append(cn)
1401             if description:
1402                 stations[icao]["description"] = ", ".join(description)
1403             lat = line[55:60].strip()
1404             if lat:
1405                 lat = int(lat)/100.0
1406                 lon = line[61:67].strip()
1407                 if lon:
1408                     lon = int(lon)/100.0
1409                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1410         count += 1
1411     metartbl.close()
1412     print("done (%s lines)." % count)
1413     message = "Reading %s..." % nsdcccc_fn
1414     sys.stdout.write(message)
1415     sys.stdout.flush()
1416     count = 0
1417     nsdcccc = codecs.open(nsdcccc_fn, "rU", "latin1")
1418     for line in nsdcccc:
1419         line = str(line)
1420         fields = line.split(";")
1421         icao = fields[0].strip().lower()
1422         if icao in stations:
1423             description = []
1424             name = " ".join( fields[3].strip().title().split() )
1425             if name: description.append(name)
1426             st = fields[4].strip()
1427             if st: description.append(st)
1428             country = " ".join( fields[5].strip().title().split() )
1429             if country: description.append(country)
1430             if description:
1431                 stations[icao]["description"] = ", ".join(description)
1432             lat, lon = fields[7:9]
1433             if lat and lon:
1434                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1435             elif "location" not in stations[icao]:
1436                 lat, lon = fields[5:7]
1437                 if lat and lon:
1438                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1439         count += 1
1440     nsdcccc.close()
1441     print("done (%s lines)." % count)
1442     message = "Reading %s..." % coopact_fn
1443     sys.stdout.write(message)
1444     sys.stdout.flush()
1445     count = 0
1446     coopact = open(coopact_fn)
1447     for line in coopact:
1448         icao = line[33:37].strip().lower()
1449         if icao in stations:
1450             iata = line[22:26].strip().lower()
1451             if len(iata) == 3: airports[iata] = { "station": icao }
1452             if "description" not in stations[icao]:
1453                 description = []
1454                 name = " ".join( line[99:129].strip().title().split() )
1455                 if name: description.append(name)
1456                 st = line[59:61].strip()
1457                 if st: description.append(st)
1458                 country = " ".join( line[38:58].strip().title().split() )
1459                 if country: description.append(country)
1460                 if description:
1461                     stations[icao]["description"] = ", ".join(description)
1462             if "location" not in stations[icao]:
1463                 lat = line[130:139].strip()
1464                 if lat:
1465                     lat = lat.replace(" ", "-")
1466                     lon = line[140:150].strip()
1467                     if lon:
1468                         lon = lon.replace(" ", "-")
1469                         stations[icao]["location"] = gecos(
1470                             "%s,%s" % (lat, lon)
1471                         )
1472         count += 1
1473     coopact.close()
1474     print("done (%s lines)." % count)
1475     message = "Reading %s..." % zlist_fn
1476     sys.stdout.write(message)
1477     sys.stdout.flush()
1478     count = 0
1479     zlist = codecs.open(zlist_fn, "rU")
1480     for line in zlist:
1481         line = line.split("#")[0].strip()
1482         if line:
1483             zones[line] = {}
1484             count += 1
1485     zlist.close()
1486     print("done (%s lines)." % count)
1487     message = "Reading %s:*..." % zcatalog_an
1488     sys.stdout.write(message)
1489     sys.stdout.flush()
1490     count = 0
1491     zcatalog = tarfile.open(zcatalog_an)
1492     for entry in zcatalog.getmembers():
1493         if entry.isfile():
1494             fnmatch = re.match(
1495                 r"([a-z]+z[0-9]+)\.txt$",
1496                 os.path.basename(entry.name)
1497             )
1498             if fnmatch:
1499                 zone = fnmatch.group(1)
1500                 if zone in zones:
1501                     data = zcatalog.extractfile(entry).readlines()
1502                     description = data[0].decode("ascii").strip()
1503                     zones[zone]["description"] = description
1504                     for line in data[1:]:
1505                         line = line.decode("latin1").strip()
1506                         urimatch = re.match("/webdocs/(.+):(.+) for ", line)
1507                         if urimatch:
1508                             uritype = urimatch.group(2).lower().replace(" ","_")
1509                             zones[zone][uritype] \
1510                                 = "http://weather.noaa.gov/%s" \
1511                                 % urimatch.group(1)
1512         count += 1
1513     zcatalog.close()
1514     print("done (%s files)." % count)
1515     message = "Reading %s..." % cpfzcf_fn
1516     sys.stdout.write(message)
1517     sys.stdout.flush()
1518     count = 0
1519     cpfz = {}
1520     cpfzcf = open(cpfzcf_fn)
1521     for line in cpfzcf:
1522         fields = line.split("|")
1523         if len(fields) == 11 \
1524             and fields[0] and fields[1] and fields[9] and fields[10]:
1525             zone = "z".join( fields[:2] ).lower()
1526             if zone in zones:
1527                 zones[zone]["centroid"] = gecos( ",".join( fields[9:] ) )
1528             elif fields[6]:
1529                 state = fields[0]
1530                 description = fields[3]
1531                 county = fields[5]
1532                 fips = "fips%s"%fields[6]
1533                 possible = [
1534                     "%s, %s" % (county, state),
1535                     "%s County, %s" % (county, state),
1536                 ]
1537                 if description.endswith(" Counties"):
1538                     description = description[:-9]
1539                 for addition in description.split(" and "):
1540                     possible.append( "%s, %s" % (addition, state) )
1541                     possible.append( "%s County, %s" % (addition, state) )
1542                 if fips in places and "centroid" in places[fips]:
1543                     for candidate in zones:
1544                         if "centroid" not in zones[candidate] and \
1545                             "description" in zones[candidate] and \
1546                             zones[candidate]["description"] in possible:
1547                             zones[candidate]["centroid"] = \
1548                                 places[fips]["centroid"]
1549         count += 1
1550     cpfzcf.close()
1551     print("done (%s lines)." % count)
1552     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1553     sys.stdout.write(message)
1554     sys.stdout.flush()
1555     count = 0
1556     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU")
1557     for line in gzcta:
1558         fields = line.decode("latin1").strip().split("\t")
1559         if len(fields) == 7 and fields[0] != "GEOID":
1560             zcta = fields[0]
1561             if zcta not in zctas: zctas[zcta] = {}
1562             zctas[zcta]["centroid"] = gecos(
1563                 ",".join( ( fields[6], fields[5] ) )
1564             )
1565             count += 1
1566     gzcta.close()
1567     print("done (%s lines)." % count)
1568     message = "Reading %s..." % overrides_fn
1569     sys.stdout.write(message)
1570     sys.stdout.flush()
1571     count = 0
1572     added = 0
1573     removed = 0
1574     changed = 0
1575     overrides = configparser.ConfigParser()
1576     overrides.readfp( codecs.open(overrides_fn, "r", "utf8") )
1577     overrideslog = []
1578     for section in overrides.sections():
1579         addopt = 0
1580         chgopt = 0
1581         if section.startswith("-"):
1582             section = section[1:]
1583             delete = True
1584         else: delete = False
1585         if re.match("[A-Za-z]{3}$", section):
1586             if delete:
1587                 if section in airports:
1588                     del( airports[section] )
1589                     logact = "removed airport %s" % section
1590                     removed += 1
1591                 else:
1592                     logact = "tried to remove nonexistent airport %s" % section
1593             else:
1594                 if section in airports:
1595                     logact = "changed airport %s" % section
1596                     changed += 1
1597                 else:
1598                     airports[section] = {}
1599                     logact = "added airport %s" % section
1600                     added += 1
1601                 for key,value in overrides.items(section):
1602                     if key in airports[section]: chgopt += 1
1603                     else: addopt += 1
1604                     if key in ("centroid", "location"):
1605                         airports[section][key] = eval(value)
1606                     else:
1607                         airports[section][key] = value
1608                 if addopt and chgopt:
1609                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1610                 elif addopt: logact += " (+%s options)" % addopt
1611                 elif chgopt: logact += " (!%s options)" % chgopt
1612         elif re.match("[A-Za-z0-9]{4}$", section):
1613             if delete:
1614                 if section in stations:
1615                     del( stations[section] )
1616                     logact = "removed station %s" % section
1617                     removed += 1
1618                 else:
1619                     logact = "tried to remove nonexistent station %s" % section
1620             else:
1621                 if section in stations:
1622                     logact = "changed station %s" % section
1623                     changed += 1
1624                 else:
1625                     stations[section] = {}
1626                     logact = "added station %s" % section
1627                     added += 1
1628                 for key,value in overrides.items(section):
1629                     if key in stations[section]: chgopt += 1
1630                     else: addopt += 1
1631                     if key in ("centroid", "location"):
1632                         stations[section][key] = eval(value)
1633                     else:
1634                         stations[section][key] = value
1635                 if addopt and chgopt:
1636                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1637                 elif addopt: logact += " (+%s options)" % addopt
1638                 elif chgopt: logact += " (!%s options)" % chgopt
1639         elif re.match("[0-9]{5}$", section):
1640             if delete:
1641                 if section in zctas:
1642                     del( zctas[section] )
1643                     logact = "removed zcta %s" % section
1644                     removed += 1
1645                 else:
1646                     logact = "tried to remove nonexistent zcta %s" % section
1647             else:
1648                 if section in zctas:
1649                     logact = "changed zcta %s" % section
1650                     changed += 1
1651                 else:
1652                     zctas[section] = {}
1653                     logact = "added zcta %s" % section
1654                     added += 1
1655                 for key,value in overrides.items(section):
1656                     if key in zctas[section]: chgopt += 1
1657                     else: addopt += 1
1658                     if key in ("centroid", "location"):
1659                         zctas[section][key] = eval(value)
1660                     else:
1661                         zctas[section][key] = value
1662                 if addopt and chgopt:
1663                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1664                 elif addopt: logact += " (+%s options)" % addopt
1665                 elif chgopt: logact += " (!%s options)" % chgopt
1666         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1667             if delete:
1668                 if section in zones:
1669                     del( zones[section] )
1670                     logact = "removed zone %s" % section
1671                     removed += 1
1672                 else:
1673                     logact = "tried to remove nonexistent zone %s" % section
1674             else:
1675                 if section in zones:
1676                     logact = "changed zone %s" % section
1677                     changed += 1
1678                 else:
1679                     zones[section] = {}
1680                     logact = "added zone %s" % section
1681                     added += 1
1682                 for key,value in overrides.items(section):
1683                     if key in zones[section]: chgopt += 1
1684                     else: addopt += 1
1685                     if key in ("centroid", "location"):
1686                         zones[section][key] = eval(value)
1687                     else:
1688                         zones[section][key] = value
1689                 if addopt and chgopt:
1690                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1691                 elif addopt: logact += " (+%s options)" % addopt
1692                 elif chgopt: logact += " (!%s options)" % chgopt
1693         elif re.match("fips[0-9]+$", section):
1694             if delete:
1695                 if section in places:
1696                     del( places[section] )
1697                     logact = "removed place %s" % section
1698                     removed += 1
1699                 else:
1700                     logact = "tried to remove nonexistent place %s" % section
1701             else:
1702                 if section in places:
1703                     logact = "changed place %s" % section
1704                     changed += 1
1705                 else:
1706                     places[section] = {}
1707                     logact = "added place %s" % section
1708                     added += 1
1709                 for key,value in overrides.items(section):
1710                     if key in places[section]: chgopt += 1
1711                     else: addopt += 1
1712                     if key in ("centroid", "location"):
1713                         places[section][key] = eval(value)
1714                     else:
1715                         places[section][key] = value
1716                 if addopt and chgopt:
1717                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1718                 elif addopt: logact += " (+%s options)" % addopt
1719                 elif chgopt: logact += " (!%s options)" % chgopt
1720         count += 1
1721         overrideslog.append("%s\n" % logact)
1722     overrideslog.sort()
1723     if os.path.exists(overrideslog_fn):
1724         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1725     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1726     overrideslog_fd.writelines(overrideslog)
1727     overrideslog_fd.close()
1728     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1729         count,
1730         added,
1731         removed,
1732         changed
1733     ) )
1734     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1735     print(
1736         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1737             estimate
1738     )
1739     count = 0
1740     milestones = list( range(51) )
1741     message = "   "
1742     sys.stdout.write(message)
1743     sys.stdout.flush()
1744     for fips in places:
1745         centroid = places[fips]["centroid"]
1746         if centroid:
1747             station = closest(centroid, stations, "location", 0.1)
1748         if station[0]:
1749             places[fips]["station"] = station
1750             count += 1
1751             if not count%100:
1752                 level = int(50*count/estimate)
1753                 if level in milestones:
1754                     for remaining in milestones[:milestones.index(level)+1]:
1755                         if remaining%5:
1756                             message = "."
1757                             sys.stdout.write(message)
1758                             sys.stdout.flush()
1759                         else:
1760                             message = "%s%%" % (remaining*2,)
1761                             sys.stdout.write(message)
1762                             sys.stdout.flush()
1763                         milestones.remove(remaining)
1764         if centroid:
1765             zone = closest(centroid, zones, "centroid", 0.1)
1766         if zone[0]:
1767             places[fips]["zone"] = zone
1768             count += 1
1769             if not count%100:
1770                 level = int(50*count/estimate)
1771                 if level in milestones:
1772                     for remaining in milestones[:milestones.index(level)+1]:
1773                         if remaining%5:
1774                             message = "."
1775                             sys.stdout.write(message)
1776                             sys.stdout.flush()
1777                         else:
1778                             message = "%s%%" % (remaining*2,)
1779                             sys.stdout.write(message)
1780                             sys.stdout.flush()
1781                         milestones.remove(remaining)
1782     for station in stations:
1783         if "location" in stations[station]:
1784             location = stations[station]["location"]
1785             if location:
1786                 zone = closest(location, zones, "centroid", 0.1)
1787             if zone[0]:
1788                 stations[station]["zone"] = zone
1789                 count += 1
1790                 if not count%100:
1791                     level = int(50*count/estimate)
1792                     if level in milestones:
1793                         for remaining in milestones[:milestones.index(level)+1]:
1794                             if remaining%5:
1795                                 message = "."
1796                                 sys.stdout.write(message)
1797                                 sys.stdout.flush()
1798                             else:
1799                                 message = "%s%%" % (remaining*2,)
1800                                 sys.stdout.write(message)
1801                                 sys.stdout.flush()
1802                             milestones.remove(remaining)
1803     for zcta in zctas.keys():
1804         centroid = zctas[zcta]["centroid"]
1805         if centroid:
1806             station = closest(centroid, stations, "location", 0.1)
1807         if station[0]:
1808             zctas[zcta]["station"] = station
1809             count += 1
1810             if not count%100:
1811                 level = int(50*count/estimate)
1812                 if level in milestones:
1813                     for remaining in milestones[ : milestones.index(level)+1 ]:
1814                         if remaining%5:
1815                             message = "."
1816                             sys.stdout.write(message)
1817                             sys.stdout.flush()
1818                         else:
1819                             message = "%s%%" % (remaining*2,)
1820                             sys.stdout.write(message)
1821                             sys.stdout.flush()
1822                         milestones.remove(remaining)
1823         if centroid:
1824             zone = closest(centroid, zones, "centroid", 0.1)
1825         if zone[0]:
1826             zctas[zcta]["zone"] = zone
1827             count += 1
1828             if not count%100:
1829                 level = int(50*count/estimate)
1830                 if level in milestones:
1831                     for remaining in milestones[:milestones.index(level)+1]:
1832                         if remaining%5:
1833                             message = "."
1834                             sys.stdout.write(message)
1835                             sys.stdout.flush()
1836                         else:
1837                             message = "%s%%" % (remaining*2,)
1838                             sys.stdout.write(message)
1839                             sys.stdout.flush()
1840                         milestones.remove(remaining)
1841     for zone in zones.keys():
1842         if "centroid" in zones[zone]:
1843             centroid = zones[zone]["centroid"]
1844             if centroid:
1845                 station = closest(centroid, stations, "location", 0.1)
1846             if station[0]:
1847                 zones[zone]["station"] = station
1848                 count += 1
1849                 if not count%100:
1850                     level = int(50*count/estimate)
1851                     if level in milestones:
1852                         for remaining in milestones[:milestones.index(level)+1]:
1853                             if remaining%5:
1854                                 message = "."
1855                                 sys.stdout.write(message)
1856                                 sys.stdout.flush()
1857                             else:
1858                                 message = "%s%%" % (remaining*2,)
1859                                 sys.stdout.write(message)
1860                                 sys.stdout.flush()
1861                             milestones.remove(remaining)
1862     for remaining in milestones:
1863         if remaining%5:
1864             message = "."
1865             sys.stdout.write(message)
1866             sys.stdout.flush()
1867         else:
1868             message = "%s%%" % (remaining*2,)
1869             sys.stdout.write(message)
1870             sys.stdout.flush()
1871     print("\n   done (%s correlations)." % count)
1872     message = "Writing %s..." % airports_fn
1873     sys.stdout.write(message)
1874     sys.stdout.flush()
1875     count = 0
1876     if os.path.exists(airports_fn):
1877         os.rename(airports_fn, "%s_old"%airports_fn)
1878     airports_fd = codecs.open(airports_fn, "w", "utf8")
1879     airports_fd.write(header)
1880     for airport in sorted( airports.keys() ):
1881         airports_fd.write("\n\n[%s]" % airport)
1882         for key, value in sorted( airports[airport].items() ):
1883             airports_fd.write( "\n%s = %s" % (key, value) )
1884         count += 1
1885     airports_fd.close()
1886     print("done (%s sections)." % count)
1887     message = "Writing %s..." % places_fn
1888     sys.stdout.write(message)
1889     sys.stdout.flush()
1890     count = 0
1891     if os.path.exists(places_fn):
1892         os.rename(places_fn, "%s_old"%places_fn)
1893     places_fd = codecs.open(places_fn, "w", "utf8")
1894     places_fd.write(header)
1895     for fips in sorted( places.keys() ):
1896         places_fd.write("\n\n[%s]" % fips)
1897         for key, value in sorted( places[fips].items() ):
1898             places_fd.write( "\n%s = %s" % (key, value) )
1899         count += 1
1900     places_fd.close()
1901     print("done (%s sections)." % count)
1902     message = "Writing %s..." % stations_fn
1903     sys.stdout.write(message)
1904     sys.stdout.flush()
1905     count = 0
1906     if os.path.exists(stations_fn):
1907         os.rename(stations_fn, "%s_old"%stations_fn)
1908     stations_fd = codecs.open(stations_fn, "w", "utf8")
1909     stations_fd.write(header)
1910     for station in sorted( stations.keys() ):
1911         stations_fd.write("\n\n[%s]" % station)
1912         for key, value in sorted( stations[station].items() ):
1913             stations_fd.write( "\n%s = %s" % (key, value) )
1914         count += 1
1915     stations_fd.close()
1916     print("done (%s sections)." % count)
1917     message = "Writing %s..." % zctas_fn
1918     sys.stdout.write(message)
1919     sys.stdout.flush()
1920     count = 0
1921     if os.path.exists(zctas_fn):
1922         os.rename(zctas_fn, "%s_old"%zctas_fn)
1923     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
1924     zctas_fd.write(header)
1925     for zcta in sorted( zctas.keys() ):
1926         zctas_fd.write("\n\n[%s]" % zcta)
1927         for key, value in sorted( zctas[zcta].items() ):
1928             zctas_fd.write( "\n%s = %s" % (key, value) )
1929         count += 1
1930     zctas_fd.close()
1931     print("done (%s sections)." % count)
1932     message = "Writing %s..." % zones_fn
1933     sys.stdout.write(message)
1934     sys.stdout.flush()
1935     count = 0
1936     if os.path.exists(zones_fn):
1937         os.rename(zones_fn, "%s_old"%zones_fn)
1938     zones_fd = codecs.open(zones_fn, "w", "utf8")
1939     zones_fd.write(header)
1940     for zone in sorted( zones.keys() ):
1941         zones_fd.write("\n\n[%s]" % zone)
1942         for key, value in sorted( zones[zone].items() ):
1943             zones_fd.write( "\n%s = %s" % (key, value) )
1944         count += 1
1945     zones_fd.close()
1946     print("done (%s sections)." % count)
1947     message = "Starting QA check..."
1948     sys.stdout.write(message)
1949     sys.stdout.flush()
1950     airports = configparser.ConfigParser()
1951     airports.read(airports_fn)
1952     places = configparser.ConfigParser()
1953     places.read(places_fn)
1954     stations = configparser.ConfigParser()
1955     stations.read(stations_fn)
1956     zctas = configparser.ConfigParser()
1957     zctas.read(zctas_fn)
1958     zones = configparser.ConfigParser()
1959     zones.read(zones_fn)
1960     qalog = []
1961     places_nocentroid = 0
1962     places_nodescription = 0
1963     for place in sorted( places.sections() ):
1964         if not places.has_option(place, "centroid"):
1965             qalog.append("%s: no centroid\n" % place)
1966             places_nocentroid += 1
1967         if not places.has_option(place, "description"):
1968             qalog.append("%s: no description\n" % place)
1969             places_nodescription += 1
1970     stations_nodescription = 0
1971     stations_nolocation = 0
1972     stations_nometar = 0
1973     for station in sorted( stations.sections() ):
1974         if not stations.has_option(station, "description"):
1975             qalog.append("%s: no description\n" % station)
1976             stations_nodescription += 1
1977         if not stations.has_option(station, "location"):
1978             qalog.append("%s: no location\n" % station)
1979             stations_nolocation += 1
1980         if not stations.has_option(station, "metar"):
1981             qalog.append("%s: no metar\n" % station)
1982             stations_nometar += 1
1983     airports_badstation = 0
1984     airports_nostation = 0
1985     for airport in sorted( airports.sections() ):
1986         if not airports.has_option(airport, "station"):
1987             qalog.append("%s: no station\n" % airport)
1988             airports_nostation += 1
1989         else:
1990             station = airports.get(airport, "station")
1991             if station not in stations.sections():
1992                 qalog.append( "%s: bad station %s\n" % (airport, station) )
1993                 airports_badstation += 1
1994     zctas_nocentroid = 0
1995     for zcta in sorted( zctas.sections() ):
1996         if not zctas.has_option(zcta, "centroid"):
1997             qalog.append("%s: no centroid\n" % zcta)
1998             zctas_nocentroid += 1
1999     zones_nocentroid = 0
2000     zones_nodescription = 0
2001     zones_noforecast = 0
2002     zones_overlapping = 0
2003     zonetable = {}
2004     for zone in zones.sections():
2005         if zones.has_option(zone, "centroid"):
2006             zonetable[zone] = {
2007                 "centroid": eval( zones.get(zone, "centroid") )
2008             }
2009     for zone in sorted( zones.sections() ):
2010         if zones.has_option(zone, "centroid"):
2011             zonetable_local = zonetable.copy()
2012             del( zonetable_local[zone] )
2013             centroid = eval( zones.get(zone, "centroid") )
2014             if centroid:
2015                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2016             if nearest[1]*radian_to_km < 1:
2017                 qalog.append( "%s: within one km of %s\n" % (
2018                     zone,
2019                     nearest[0]
2020                 ) )
2021                 zones_overlapping += 1
2022         else:
2023             qalog.append("%s: no centroid\n" % zone)
2024             zones_nocentroid += 1
2025         if not zones.has_option(zone, "description"):
2026             qalog.append("%s: no description\n" % zone)
2027             zones_nodescription += 1
2028         if not zones.has_option(zone, "zone_forecast"):
2029             qalog.append("%s: no forecast\n" % zone)
2030             zones_noforecast += 1
2031     if os.path.exists(qalog_fn):
2032         os.rename(qalog_fn, "%s_old"%qalog_fn)
2033     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2034     qalog_fd.writelines(qalog)
2035     qalog_fd.close()
2036     if qalog:
2037         print("issues found (see %s for details):"%qalog_fn)
2038         if airports_badstation:
2039             print("   %s airports with invalid station"%airports_badstation)
2040         if airports_nostation:
2041             print("   %s airports with no station"%airports_nostation)
2042         if places_nocentroid:
2043             print("   %s places with no centroid"%places_nocentroid)
2044         if places_nodescription:
2045             print("   %s places with no description"%places_nodescription)
2046         if stations_nodescription:
2047             print("   %s stations with no description"%stations_nodescription)
2048         if stations_nolocation:
2049             print("   %s stations with no location"%stations_nolocation)
2050         if stations_nometar:
2051             print("   %s stations with no METAR"%stations_nometar)
2052         if zctas_nocentroid:
2053             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2054         if zones_nocentroid:
2055             print("   %s zones with no centroid"%zones_nocentroid)
2056         if zones_nodescription:
2057             print("   %s zones with no description"%zones_nodescription)
2058         if zones_noforecast:
2059             print("   %s zones with no forecast"%zones_noforecast)
2060         if zones_overlapping:
2061             print("   %s zones within one km of another"%zones_overlapping)
2062     else: print("no issues found.")
2063     print("Indexing complete!")