"""Contains various object definitions needed by the weather utility.""" weather_copyright = """\ # Copyright (c) 2006-2016 Jeremy Stanley . Permission to # use, copy, modify, and distribute this software is granted under terms # provided in the LICENSE file distributed with this software. #""" weather_version = "2.3" radian_to_km = 6372.795484 radian_to_mi = 3959.871528 def pyversion(ref=None): """Determine the Python version and optionally compare to a reference.""" import platform ver = platform.python_version() if ref: return [ int(x) for x in ver.split(".")[:2] ] >= [ int(x) for x in ref.split(".")[:2] ] else: return ver class Selections: """An object to contain selection data.""" def __init__(self): """Store the config, options and arguments.""" self.config = get_config() self.options, self.arguments = get_options(self.config) if self.get_bool("cache") and self.get_bool("cache_search") \ and not self.get_bool("longlist"): integrate_search_cache( self.config, self.get("cachedir"), self.get("setpath") ) if not self.arguments: if "id" in self.options.__dict__ \ and self.options.__dict__["id"]: self.arguments.append( self.options.__dict__["id"] ) del( self.options.__dict__["id"] ) import sys message = "WARNING: the --id option is deprecated and will eventually be removed\n" sys.stderr.write(message) elif "city" in self.options.__dict__ \ and self.options.__dict__["city"] \ and "st" in self.options.__dict__ \ and self.options.__dict__["st"]: self.arguments.append( "^%s city, %s" % ( self.options.__dict__["city"], self.options.__dict__["st"] ) ) del( self.options.__dict__["city"] ) del( self.options.__dict__["st"] ) import sys message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n" sys.stderr.write(message) def get(self, option, argument=None): """Retrieve data from the config or options.""" if argument: if self.config.has_section(argument) and ( self.config.has_option(argument, "city") \ or self.config.has_option(argument, "id") \ or self.config.has_option(argument, "st") ): self.config.remove_section(argument) import sys message = "WARNING: the city/id/st options are now unsupported in aliases\n" sys.stderr.write(message) if not self.config.has_section(argument): guessed = guess( argument, path=self.get("setpath"), info=self.get("info"), cache_search=( self.get("cache") and self.get("cache_search") ), cachedir=self.get("cachedir"), quiet=self.get_bool("quiet") ) self.config.add_section(argument) for item in guessed.items(): self.config.set(argument, *item) if self.config.has_option(argument, option): return self.config.get(argument, option) if option in self.options.__dict__: return self.options.__dict__[option] else: import os, sys message = "%s error: no URI defined for %s\n" % ( os.path.basename( sys.argv[0] ), option ) sys.stderr.write(message) exit(1) def get_bool(self, option, argument=None): """Get data and coerce to a boolean if necessary.""" return bool(self.get(option, argument)) def getint(self, option, argument=None): """Get data and coerce to an integer if necessary.""" value = self.get(option, argument) if value: return int(value) else: return 0 def average(coords): """Average a list of coordinates.""" x = 0 y = 0 for coord in coords: x += coord[0] y += coord[1] count = len(coords) return (x/count, y/count) def filter_units(line, units="imperial"): """Filter or convert units in a line of text between US/UK and metric.""" import re # filter lines with both pressures in the form of "X inches (Y hPa)" or # "X in. Hg (Y hPa)" dual_p = re.match( "(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)", line ) if dual_p: preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups() if units == "imperial": line = preamble + in_hg + trailer elif units == "metric": line = preamble + hpa + trailer # filter lines with both temperatures in the form of "X F (Y C)" dual_t = re.match( "(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)", line ) if dual_t: preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups() if units == "imperial": line = preamble + fahrenheit + trailer elif units == "metric": line = preamble + celsius + trailer # if metric is desired, convert distances in the form of "X mile(s)" to # "Y kilometer(s)" if units == "metric": imperial_d = re.match( "(.* )(\d+)( mile\(s\))(.*)", line ) if imperial_d: preamble, mi, m_u, trailer = imperial_d.groups() line = preamble + str(int(round(int(mi)*1.609344))) \ + " kilometer(s)" + trailer # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is # desired, convert to "Z KPH" imperial_s = re.match( "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)", line ) if imperial_s: preamble, mph, m_u, kt, trailer = imperial_s.groups() if units == "imperial": line = preamble + mph + m_u + trailer elif units == "metric": line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \ trailer imperial_s = re.match( "(.* )(\d+)( MPH)( \(\d+ KT\))(.*)", line ) if imperial_s: preamble, mph, m_u, kt, trailer = imperial_s.groups() if units == "imperial": line = preamble + mph + m_u + trailer elif units == "metric": line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \ trailer # if imperial is desired, qualify given forcast temperatures like "X F"; if # metric is desired, convert to "Y C" imperial_t = re.match( "(.* )(High |high |Low |low )(\d+)(\.|,)(.*)", line ) if imperial_t: preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups() if units == "imperial": line = preamble + parameter + fahrenheit + " F" + sep + trailer elif units == "metric": line = preamble + parameter \ + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \ + trailer # hand off the resulting line return line def get_uri( uri, ignore_fail=False, cache_data=False, cacheage=900, cachedir="." ): """Return a string containing the results of a URI GET.""" if pyversion("3"): import urllib, urllib.error, urllib.request URLError = urllib.error.URLError urlopen = urllib.request.urlopen else: import urllib2 as urllib URLError = urllib.URLError urlopen = urllib.urlopen import os, time if cache_data: dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" ) if not os.path.exists(dcachedir): try: os.makedirs(dcachedir) except (IOError, OSError): pass dcache_fn = os.path.join( dcachedir, uri.split(":")[1].replace("/","_") ) now = time.time() if cache_data and os.access(dcache_fn, os.R_OK) \ and now-cacheage < os.stat(dcache_fn).st_mtime <= now: dcache_fd = open(dcache_fn) data = dcache_fd.read() dcache_fd.close() else: try: if pyversion("3"): data = urlopen(uri).read().decode("utf-8") else: data = urlopen(uri).read() except URLError: if ignore_fail: return "" else: import os, sys, traceback message = "%s error: failed to retrieve\n %s\n %s" % ( os.path.basename( sys.argv[0] ), uri, traceback.format_exception_only( sys.exc_type, sys.exc_value )[0] ) sys.stderr.write(message) sys.exit(1) if cache_data: try: import codecs dcache_fd = codecs.open(dcache_fn, "w", "utf-8") dcache_fd.write(data) dcache_fd.close() except (IOError, OSError): pass return data def get_metar( uri=None, verbose=False, quiet=False, headers=None, imperial=False, metric=False, cache_data=False, cacheage=900, cachedir="." ): """Return a summarized METAR for the specified station.""" if not uri: import os, sys message = "%s error: METAR URI required for conditions\n" % \ os.path.basename( sys.argv[0] ) sys.stderr.write(message) sys.exit(1) metar = get_uri( uri, cache_data=cache_data, cacheage=cacheage, cachedir=cachedir ) if pyversion("3") and type(metar) is bytes: metar = metar.decode("utf-8") if verbose: return metar else: import re lines = metar.split("\n") if not headers: headers = \ "relative_humidity," \ + "precipitation_last_hour," \ + "sky conditions," \ + "temperature," \ + "heat index," \ + "windchill," \ + "weather," \ + "wind" headerlist = headers.lower().replace("_"," ").split(",") output = [] if not quiet: title = "Current conditions at %s" place = lines[0].split(", ") if len(place) > 1: place = "%s, %s" % ( place[0].title(), place[1] ) else: place = "" output.append(title%place) output.append("Last updated " + lines[1]) header_match = False for header in headerlist: for line in lines: if line.lower().startswith(header + ":"): if re.match(r".*:\d+$", line): line = line[:line.rfind(":")] if imperial: line = filter_units(line, units="imperial") elif metric: line = filter_units(line, units="metric") if quiet: output.append(line) else: output.append(" " + line) header_match = True if not header_match: output.append( "(no conditions matched your header list, try with --verbose)" ) return "\n".join(output) def get_alert( uri=None, verbose=False, quiet=False, cache_data=False, cacheage=900, cachedir="." ): """Return alert notice for the specified URI.""" if not uri: import os, sys message = "%s error: Alert URI required for alerts\n" % \ os.path.basename( sys.argv[0] ) sys.stderr.write(message) sys.exit(1) alert = get_uri( uri, ignore_fail=True, cache_data=cache_data, cacheage=cacheage, cachedir=cachedir ).strip() if pyversion("3") and type(alert) is bytes: alert = alert.decode("utf-8") if alert: if verbose: return alert else: if alert.find("\nNATIONAL WEATHER SERVICE") == -1: muted = False else: muted = True lines = alert.split("\n") import time valid_time = time.strftime("%Y%m%d%H%M") output = [] for line in lines: if line.startswith("Expires:") \ and "Expires:" + valid_time > line: return "" if muted and line.startswith("NATIONAL WEATHER SERVICE"): muted = False line = "" elif line == "&&": line = "" elif line == "$$": muted = True if line and not muted: if quiet: output.append(line) else: output.append(" " + line) return "\n".join(output) def get_options(config): """Parse the options passed on the command line.""" # for optparse's builtin -h/--help option usage = \ "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]" # for optparse's builtin --version option verstring = "%prog " + weather_version # create the parser import optparse option_parser = optparse.OptionParser(usage=usage, version=verstring) # separate options object from list of arguments and return both # the -a/--alert option if config.has_option("default", "alert"): default_alert = bool(config.get("default", "alert")) else: default_alert = False option_parser.add_option("-a", "--alert", dest="alert", action="store_true", default=default_alert, help="include local alert notices") # the --atypes option if config.has_option("default", "atypes"): default_atypes = config.get("default", "atypes") else: default_atypes = \ "coastal_flood_statement," \ + "flash_flood_statement," \ + "flash_flood_warning," \ + "flash_flood_watch," \ + "flood_statement," \ + "flood_warning," \ + "marine_weather_statement," \ + "river_statement," \ + "severe_thunderstorm_warning," \ + "severe_weather_statement," \ + "short_term_forecast," \ + "special_marine_warning," \ + "special_weather_statement," \ + "tornado_warning," \ + "urgent_weather_message" option_parser.add_option("--atypes", dest="atypes", default=default_atypes, help="list of alert notification types to display") # the --build-sets option option_parser.add_option("--build-sets", dest="build_sets", action="store_true", default=False, help="(re)build location correlation sets") # the --cacheage option if config.has_option("default", "cacheage"): default_cacheage = config.getint("default", "cacheage") else: default_cacheage = 900 option_parser.add_option("--cacheage", dest="cacheage", default=default_cacheage, help="duration in seconds to refresh cached data") # the --cachedir option if config.has_option("default", "cachedir"): default_cachedir = config.get("default", "cachedir") else: default_cachedir = "~/.weather" option_parser.add_option("--cachedir", dest="cachedir", default=default_cachedir, help="directory for storing cached searches and data") # the -f/--forecast option if config.has_option("default", "forecast"): default_forecast = bool(config.get("default", "forecast")) else: default_forecast = False option_parser.add_option("-f", "--forecast", dest="forecast", action="store_true", default=default_forecast, help="include a local forecast") # the --headers option if config.has_option("default", "headers"): default_headers = config.get("default", "headers") else: default_headers = \ "temperature," \ + "relative_humidity," \ + "wind," \ + "heat_index," \ + "windchill," \ + "weather," \ + "sky_conditions," \ + "precipitation_last_hour" option_parser.add_option("--headers", dest="headers", default=default_headers, help="list of conditions headers to display") # the --imperial option if config.has_option("default", "imperial"): default_imperial = bool(config.get("default", "imperial")) else: default_imperial = False option_parser.add_option("--imperial", dest="imperial", action="store_true", default=default_imperial, help="filter/convert conditions for US/UK units") # the --info option option_parser.add_option("--info", dest="info", action="store_true", default=False, help="output detailed information for your search") # the -l/--list option option_parser.add_option("-l", "--list", dest="list", action="store_true", default=False, help="list all configured aliases and cached searches") # the --longlist option option_parser.add_option("--longlist", dest="longlist", action="store_true", default=False, help="display details of all configured aliases") # the -m/--metric option if config.has_option("default", "metric"): default_metric = bool(config.get("default", "metric")) else: default_metric = False option_parser.add_option("-m", "--metric", dest="metric", action="store_true", default=default_metric, help="filter/convert conditions for metric units") # the -n/--no-conditions option if config.has_option("default", "conditions"): default_conditions = bool(config.get("default", "conditions")) else: default_conditions = True option_parser.add_option("-n", "--no-conditions", dest="conditions", action="store_false", default=default_conditions, help="disable output of current conditions") # the --no-cache option if config.has_option("default", "cache"): default_cache = bool(config.get("default", "cache")) else: default_cache = True option_parser.add_option("--no-cache", dest="cache", action="store_false", default=True, help="disable all caching (searches and data)") # the --no-cache-data option if config.has_option("default", "cache_data"): default_cache_data = bool(config.get("default", "cache_data")) else: default_cache_data = True option_parser.add_option("--no-cache-data", dest="cache_data", action="store_false", default=True, help="disable retrieved data caching") # the --no-cache-search option if config.has_option("default", "cache_search"): default_cache_search = bool(config.get("default", "cache_search")) else: default_cache_search = True option_parser.add_option("--no-cache-search", dest="cache_search", action="store_false", default=True, help="disable search result caching") # the -q/--quiet option if config.has_option("default", "quiet"): default_quiet = bool(config.get("default", "quiet")) else: default_quiet = False option_parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=default_quiet, help="skip preambles and don't indent") # the --setpath option if config.has_option("default", "setpath"): default_setpath = config.get("default", "setpath") else: default_setpath = ".:~/.weather" option_parser.add_option("--setpath", dest="setpath", default=default_setpath, help="directory search path for correlation sets") # the -v/--verbose option if config.has_option("default", "verbose"): default_verbose = bool(config.get("default", "verbose")) else: default_verbose = False option_parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=default_verbose, help="show full decoded feeds") # deprecated options if config.has_option("default", "city"): default_city = config.get("default", "city") else: default_city = "" option_parser.add_option("-c", "--city", dest="city", default=default_city, help=optparse.SUPPRESS_HELP) if config.has_option("default", "id"): default_id = config.get("default", "id") else: default_id = "" option_parser.add_option("-i", "--id", dest="id", default=default_id, help=optparse.SUPPRESS_HELP) if config.has_option("default", "st"): default_st = config.get("default", "st") else: default_st = "" option_parser.add_option("-s", "--st", dest="st", default=default_st, help=optparse.SUPPRESS_HELP) options, arguments = option_parser.parse_args() return options, arguments def get_config(): """Parse the aliases and configuration.""" if pyversion("3"): import configparser else: import ConfigParser as configparser config = configparser.ConfigParser() import os rcfiles = [ "/etc/weatherrc", "/etc/weather/weatherrc", os.path.expanduser("~/.weather/weatherrc"), os.path.expanduser("~/.weatherrc"), "weatherrc" ] for rcfile in rcfiles: if os.access(rcfile, os.R_OK): config.read(rcfile) for section in config.sections(): if section != section.lower(): if config.has_section(section.lower()): config.remove_section(section.lower()) config.add_section(section.lower()) for option,value in config.items(section): config.set(section.lower(), option, value) return config def integrate_search_cache(config, cachedir, setpath): """Add cached search results into the configuration.""" if pyversion("3"): import configparser else: import ConfigParser as configparser import os, time scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" ) if not os.access(scache_fn, os.R_OK): return config scache_fd = open(scache_fn) created = float( scache_fd.readline().split(":")[1].strip().split()[0] ) scache_fd.close() now = time.time() datafiles = data_index(setpath) if datafiles: data_freshness = sorted( [ x[1] for x in datafiles.values() ], reverse=True )[0] else: data_freshness = now if created < data_freshness <= now: try: os.remove(scache_fn) print( "[clearing outdated %s]" % scache_fn ) except (IOError, OSError): pass return config scache = configparser.ConfigParser() scache.read(scache_fn) for section in scache.sections(): if not config.has_section(section): config.add_section(section) for option,value in scache.items(section): config.set(section, option, value) return config def list_aliases(config, detail=False): """Return a formatted list of aliases defined in the config.""" if detail: output = "\n# configured alias details..." for section in sorted(config.sections()): output += "\n\n[%s]" % section for item in sorted(config.items(section)): output += "\n%s = %s" % item output += "\n" else: output = "configured aliases and cached searches..." for section in sorted(config.sections()): if config.has_option(section, "description"): description = config.get(section, "description") else: description = "(no description provided)" output += "\n %s: %s" % (section, description) return output def data_index(path): import os datafiles = {} for filename in ("airports", "places", "stations", "zctas", "zones"): for dirname in path.split(":"): for extension in ("", ".gz", ".txt"): candidate = os.path.expanduser( os.path.join( dirname, "".join( (filename, extension) ) ) ) if os.path.exists(candidate): datafiles[filename] = ( candidate, os.stat(candidate).st_mtime ) break if filename in datafiles: break return datafiles def guess( expression, path=".", max_results=20, info=False, cache_search=False, cacheage=900, cachedir=".", quiet=False ): """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone.""" import codecs, datetime, time, os, re, sys if pyversion("3"): import configparser else: import ConfigParser as configparser datafiles = data_index(path) if re.match("[A-Za-z]{3}$", expression): searchtype = "airport" elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station" elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone" elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA" elif re.match( r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$", expression ): searchtype = "coordinates" elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS" else: searchtype = "name" cache_search = False if cache_search: action = "caching" else: action = "using" if info: scores = [ (0.005, "bad"), (0.025, "poor"), (0.160, "suspect"), (0.500, "mediocre"), (0.840, "good"), (0.975, "great"), (0.995, "excellent"), (1.000, "ideal"), ] if not quiet: print("Searching via %s..."%searchtype) stations = configparser.ConfigParser() dataname = "stations" if dataname in datafiles: datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip if pyversion("3"): stations.read_string( gzip.open(datafile).read().decode("utf-8") ) else: stations.readfp( gzip.open(datafile) ) else: stations.read(datafile) else: message = "%s error: can't find \"%s\" data file\n" % ( os.path.basename( sys.argv[0] ), dataname ) sys.stderr.write(message) exit(1) zones = configparser.ConfigParser() dataname = "zones" if dataname in datafiles: datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip if pyversion("3"): zones.read_string( gzip.open(datafile).read().decode("utf-8") ) else: zones.readfp( gzip.open(datafile) ) else: zones.read(datafile) else: message = "%s error: can't find \"%s\" data file\n" % ( os.path.basename( sys.argv[0] ), dataname ) sys.stderr.write(message) exit(1) search = None station = ("", 0) zone = ("", 0) dataset = None possibilities = [] uris = {} if searchtype == "airport": expression = expression.lower() airports = configparser.ConfigParser() dataname = "airports" if dataname in datafiles: datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip if pyversion("3"): airports.read_string( gzip.open(datafile).read().decode("utf-8") ) else: airports.readfp( gzip.open(datafile) ) else: airports.read(datafile) else: message = "%s error: can't find \"%s\" data file\n" % ( os.path.basename( sys.argv[0] ), dataname ) sys.stderr.write(message) exit(1) if airports.has_section(expression) \ and airports.has_option(expression, "station"): search = (expression, "IATA/FAA airport code %s" % expression) station = ( airports.get(expression, "station"), 0 ) if stations.has_option(station[0], "zone"): zone = eval( stations.get(station[0], "zone") ) dataset = stations if not ( info or quiet ) \ and stations.has_option( station[0], "description" ): print( "[%s result %s]" % ( action, stations.get(station[0], "description") ) ) else: message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % ( expression, datafiles["airports"][0] ) sys.stderr.write(message) exit(1) elif searchtype == "station": expression = expression.lower() if stations.has_section(expression): station = (expression, 0) if not search: search = (expression, "ICAO station code %s" % expression) if stations.has_option(expression, "zone"): zone = eval( stations.get(expression, "zone") ) dataset = stations if not ( info or quiet ) \ and stations.has_option(expression, "description"): print( "[%s result %s]" % ( action, stations.get(expression, "description") ) ) else: message = "No ICAO weather station \"%s\" in the %s file.\n" % ( expression, datafiles["stations"][0] ) sys.stderr.write(message) exit(1) elif searchtype == "zone": expression = expression.lower() if zones.has_section(expression) \ and zones.has_option(expression, "station"): zone = (expression, 0) station = eval( zones.get(expression, "station") ) dataset = zones search = (expression, "NWS/NOAA weather zone %s" % expression) if not ( info or quiet ) \ and zones.has_option(expression, "description"): print( "[%s result %s]" % ( action, zones.get(expression, "description") ) ) else: message = "No usable NWS weather zone \"%s\" in the %s file.\n" % ( expression, datafiles["zones"][0] ) sys.stderr.write(message) exit(1) elif searchtype == "ZCTA": zctas = configparser.ConfigParser() dataname = "zctas" if dataname in datafiles: datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip if pyversion("3"): zctas.read_string( gzip.open(datafile).read().decode("utf-8") ) else: zctas.readfp( gzip.open(datafile) ) else: zctas.read(datafile) else: message = "%s error: can't find \"%s\" data file\n" % ( os.path.basename( sys.argv[0] ), dataname ) sys.stderr.write(message) exit(1) dataset = zctas if zctas.has_section(expression) \ and zctas.has_option(expression, "station"): station = eval( zctas.get(expression, "station") ) search = (expression, "Census ZCTA (ZIP code) %s" % expression) if zctas.has_option(expression, "zone"): zone = eval( zctas.get(expression, "zone") ) else: message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % ( expression, datafiles["zctas"][0] ) sys.stderr.write(message) exit(1) elif searchtype == "coordinates": search = (expression, "Geographic coordinates %s" % expression) stationtable = {} for station in stations.sections(): if stations.has_option(station, "location"): stationtable[station] = { "location": eval( stations.get(station, "location") ) } station = closest( gecos(expression), stationtable, "location", 0.1 ) if not station[0]: message = "No ICAO weather station found near %s.\n" % expression sys.stderr.write(message) exit(1) zonetable = {} for zone in zones.sections(): if zones.has_option(zone, "centroid"): zonetable[zone] = { "centroid": eval( zones.get(zone, "centroid") ) } zone = closest( gecos(expression), zonetable, "centroid", 0.1 ) if not zone[0]: message = "No NWS weather zone near %s; forecasts unavailable.\n" \ % expression sys.stderr.write(message) elif searchtype in ("FIPS", "name"): places = configparser.ConfigParser() dataname = "places" if dataname in datafiles: datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip if pyversion("3"): places.read_string( gzip.open(datafile).read().decode("utf-8") ) else: places.readfp( gzip.open(datafile) ) else: places.read(datafile) else: message = "%s error: can't find \"%s\" data file\n" % ( os.path.basename( sys.argv[0] ), dataname ) sys.stderr.write(message) exit(1) dataset = places place = expression.lower() if places.has_section(place) and places.has_option(place, "station"): station = eval( places.get(place, "station") ) search = (expression, "Census Place %s" % expression) if places.has_option(place, "description"): search = ( search[0], search[1] + ", %s" % places.get(place, "description") ) if places.has_option(place, "zone"): zone = eval( places.get(place, "zone") ) if not ( info or quiet ) \ and places.has_option(place, "description"): print( "[%s result %s]" % ( action, places.get(place, "description") ) ) else: for place in places.sections(): if places.has_option(place, "description") \ and places.has_option(place, "station") \ and re.search( expression, places.get(place, "description"), re.I ): possibilities.append(place) for place in stations.sections(): if stations.has_option(place, "description") \ and re.search( expression, stations.get(place, "description"), re.I ): possibilities.append(place) for place in zones.sections(): if zones.has_option(place, "description") \ and zones.has_option(place, "station") \ and re.search( expression, zones.get(place, "description"), re.I ): possibilities.append(place) if len(possibilities) == 1: place = possibilities[0] if places.has_section(place): station = eval( places.get(place, "station") ) description = places.get(place, "description") if places.has_option(place, "zone"): zone = eval( places.get(place, "zone" ) ) search = ( expression, "%s: %s" % (place, description) ) elif stations.has_section(place): station = (place, 0.0) description = stations.get(place, "description") if stations.has_option(place, "zone"): zone = eval( stations.get(place, "zone" ) ) search = ( expression, "ICAO station code %s" % place ) elif zones.has_section(place): station = eval( zones.get(place, "station") ) description = zones.get(place, "description") zone = (place, 0.0) search = ( expression, "NWS/NOAA weather zone %s" % place ) if not ( info or quiet ): print( "[%s result %s]" % (action, description) ) if not possibilities and not station[0]: message = "No FIPS code/census area match in the %s file.\n" % ( datafiles["places"][0] ) sys.stderr.write(message) exit(1) if station[0]: uris["metar"] = stations.get( station[0], "metar" ) if zone[0]: for key,value in zones.items( zone[0] ): if key not in ("centroid", "description", "station"): uris[key] = value elif possibilities: count = len(possibilities) if count <= max_results: print( "Your search is ambiguous, returning %s matches:" % count ) for place in sorted(possibilities): if places.has_section(place): print( " [%s] %s" % ( place, places.get(place, "description") ) ) elif stations.has_section(place): print( " [%s] %s" % ( place, stations.get(place, "description") ) ) elif zones.has_section(place): print( " [%s] %s" % ( place, zones.get(place, "description") ) ) else: print( "Your search is too ambiguous, returning %s matches." % count ) exit(0) if info: stationlist = [] zonelist = [] if dataset: for section in dataset.sections(): if dataset.has_option(section, "station"): stationlist.append( eval( dataset.get(section, "station") )[1] ) if dataset.has_option(section, "zone"): zonelist.append( eval( dataset.get(section, "zone") )[1] ) stationlist.sort() zonelist.sort() scount = len(stationlist) zcount = len(zonelist) sranks = [] zranks = [] for score in scores: if stationlist: sranks.append( stationlist[ int( (1-score[0]) * scount ) ] ) if zonelist: zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] ) description = search[1] uris["description"] = description print( "%s\n%s" % ( description, "-" * len(description) ) ) print( "%s: %s" % ( station[0], stations.get( station[0], "description" ) ) ) km = radian_to_km*station[1] mi = radian_to_mi*station[1] if sranks and not description.startswith("ICAO station code "): for index in range(0, len(scores)): if station[1] >= sranks[index]: score = scores[index][1] break print( " (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi ) ) elif searchtype is "coordinates": print( " (%.3gkm, %.3gmi)" % (km, mi) ) if zone[0]: print( "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) ) ) km = radian_to_km*zone[1] mi = radian_to_mi*zone[1] if zranks and not description.startswith("NWS/NOAA weather zone "): for index in range(0, len(scores)): if zone[1] >= zranks[index]: score = scores[index][1] break print( " (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi ) ) elif searchtype is "coordinates" and zone[0]: print( " (%.3gkm, %.3gmi)" % (km, mi) ) if cache_search: now = time.time() nowstamp = "%s (%s)" % ( now, datetime.datetime.isoformat( datetime.datetime.fromtimestamp(now), " " ) ) search_cache = ["\n"] search_cache.append( "[%s]\n" % search[0] ) search_cache.append( "description = cached %s\n" % nowstamp ) for uriname in sorted(uris.keys()): search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) ) real_cachedir = os.path.expanduser(cachedir) if not os.path.exists(real_cachedir): try: os.makedirs(real_cachedir) except (IOError, OSError): pass scache_fn = os.path.join(real_cachedir, "searches") if not os.path.exists(scache_fn): then = sorted( [ x[1] for x in datafiles.values() ], reverse=True )[0] thenstamp = "%s (%s)" % ( then, datetime.datetime.isoformat( datetime.datetime.fromtimestamp(then), " " ) ) search_cache.insert( 0, "# based on data files from: %s\n" % thenstamp ) try: scache_existing = configparser.ConfigParser() scache_existing.read(scache_fn) if not scache_existing.has_section(search[0]): scache_fd = codecs.open(scache_fn, "a", "utf-8") scache_fd.writelines(search_cache) scache_fd.close() except (IOError, OSError): pass if not info: return(uris) def closest(position, nodes, fieldname, angle=None): import math if not angle: angle = 2*math.pi match = None for name in nodes: if fieldname in nodes[name]: node = nodes[name][fieldname] if node and abs( position[0]-node[0] ) < angle: if abs( position[1]-node[1] ) < angle \ or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle: if position == node: angle = 0 match = name else: candidate = math.acos( math.sin( position[0] ) * math.sin( node[0] ) \ + math.cos( position[0] ) \ * math.cos( node[0] ) \ * math.cos( position[1] - node[1] ) ) if candidate < angle: angle = candidate match = name if match: match = str(match) return (match, angle) def gecos(formatted): import math, re coordinates = formatted.split(",") for coordinate in range(0, 2): degrees, foo, minutes, bar, seconds, hemisphere = re.match( r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$", coordinates[coordinate].strip().lower() ).groups() value = float(degrees) if minutes: value += float(minutes)/60 if seconds: value += float(seconds)/3600 if hemisphere and hemisphere in "sw": value *= -1 coordinates[coordinate] = math.radians(value) return tuple(coordinates) def correlate(): import codecs, datetime, hashlib, os, re, sys, tarfile, time, zipfile if pyversion("3"): import configparser else: import ConfigParser as configparser gcounties_an = "2015_Gaz_counties_national.zip" gcounties_fn = "2015_Gaz_counties_national.txt" gcousubs_an = "2015_Gaz_cousubs_national.zip" gcousubs_fn = "2015_Gaz_cousubs_national.txt" gplace_an = "2015_Gaz_place_national.zip" gplace_fn = "2015_Gaz_place_national.txt" gzcta_an = "2015_Gaz_zcta_national.zip" gzcta_fn = "2015_Gaz_zcta_national.txt" for filename in os.listdir("."): if re.match("bp[0-9][0-9][a-z][a-z][0-9][0-9].dbx$", filename): cpfzcf_fn = filename break nsdcccc_fn = "nsd_cccc.txt" zcatalog_an = "zonecatalog.curr.tar" metartbl_fn = "metar.tbl" coopstn_fn = "coop-stations.txt" overrides_fn = "overrides.conf" overrideslog_fn = "overrides.log" slist_fn = "slist" zlist_fn = "zlist" qalog_fn = "qa.log" airports_fn = "airports" places_fn = "places" stations_fn = "stations" zctas_fn = "zctas" zones_fn = "zones" header = """\ %s # generated by %s on %s from these public domain sources: # # http://www.census.gov/geo/maps-data/data/gazetteer2015.html # %s %s %s # %s %s %s # %s %s %s # %s %s %s # # http://www.weather.gov/geodata/catalog/wsom/html/cntyzone.htm # %s %s %s # # http://tgftp.nws.noaa.gov/data/nsd_cccc.txt # %s %s %s # # http://tgftp.nws.noaa.gov/data/zonecatalog.curr.tar # %s %s %s # # http://www.nco.ncep.noaa.gov/pmb/codes/nwprod/dictionaries/metar.tbl # %s %s %s # # http://www.ncdc.noaa.gov/homr/reports # %s %s %s # # ...and these manually-generated or hand-compiled adjustments: # %s %s %s # %s %s %s # %s %s %s\ """ % ( weather_copyright, os.path.basename( sys.argv[0] ), datetime.date.isoformat( datetime.datetime.fromtimestamp( time.time() ) ), hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(gcounties_an) ) ), gcounties_an, hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) ) ), gcousubs_an, hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(gplace_an) ) ), gplace_an, hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) ) ), gzcta_an, hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(cpfzcf_fn) ) ), cpfzcf_fn, hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) ) ), nsdcccc_fn, hashlib.md5( open(zcatalog_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(zcatalog_an) ) ), zcatalog_an, hashlib.md5( open(metartbl_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(metartbl_fn) ) ), metartbl_fn, hashlib.md5( open(coopstn_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(coopstn_fn) ) ), coopstn_fn, hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) ) ), overrides_fn, hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(slist_fn) ) ), slist_fn, hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(zlist_fn) ) ), zlist_fn ) airports = {} places = {} stations = {} zctas = {} zones = {} message = "Reading %s:%s..." % (gcounties_an, gcounties_fn) sys.stdout.write(message) sys.stdout.flush() count = 0 gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU") columns = gcounties.readline().decode("latin1").strip().split("\t") for line in gcounties: fields = line.decode("latin1").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() f_intptlat = fields[ columns.index("INTPTLAT") ].strip() f_intptlong = fields[ columns.index("INTPTLONG") ].strip() if f_geoid and f_name and f_usps and f_intptlat and f_intptlong: fips = "fips%s" % f_geoid if fips not in places: places[fips] = {} places[fips]["centroid"] = gecos( "%s,%s" % (f_intptlat, f_intptlong) ) places[fips]["description"] = "%s, %s" % (f_name, f_usps) count += 1 gcounties.close() print("done (%s lines)." % count) message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn) sys.stdout.write(message) sys.stdout.flush() count = 0 gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU") columns = gcousubs.readline().decode("latin1").strip().split("\t") for line in gcousubs: fields = line.decode("latin1").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() f_intptlat = fields[ columns.index("INTPTLAT") ].strip() f_intptlong = fields[ columns.index("INTPTLONG") ].strip() if f_geoid and f_name and f_usps and f_intptlat and f_intptlong: fips = "fips%s" % f_geoid if fips not in places: places[fips] = {} places[fips]["centroid"] = gecos( "%s,%s" % (f_intptlat, f_intptlong) ) places[fips]["description"] = "%s, %s" % (f_name, f_usps) count += 1 gcousubs.close() print("done (%s lines)." % count) message = "Reading %s:%s..." % (gplace_an, gplace_fn) sys.stdout.write(message) sys.stdout.flush() count = 0 gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "rU") columns = gplace.readline().decode("latin1").strip().split("\t") for line in gplace: fields = line.decode("latin1").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() f_intptlat = fields[ columns.index("INTPTLAT") ].strip() f_intptlong = fields[ columns.index("INTPTLONG") ].strip() if f_geoid and f_name and f_usps and f_intptlat and f_intptlong: fips = "fips%s" % f_geoid if fips not in places: places[fips] = {} places[fips]["centroid"] = gecos( "%s,%s" % (f_intptlat, f_intptlong) ) places[fips]["description"] = "%s, %s" % (f_name, f_usps) count += 1 gplace.close() print("done (%s lines)." % count) message = "Reading %s..." % slist_fn sys.stdout.write(message) sys.stdout.flush() count = 0 slist = codecs.open(slist_fn, "rU") for line in slist: icao = line.split("#")[0].strip() if icao: stations[icao] = { "metar": "http://tgftp.nws.noaa.gov/data/observations/"\ + "metar/decoded/%s.TXT" % icao.upper() } count += 1 slist.close() print("done (%s lines)." % count) message = "Reading %s..." % metartbl_fn sys.stdout.write(message) sys.stdout.flush() count = 0 metartbl = codecs.open(metartbl_fn, "rU") for line in metartbl: icao = line[:4].strip().lower() if icao in stations: description = [] name = " ".join( line[16:48].replace("_", " ").strip().title().split() ) if name: description.append(name) st = line[49:51].strip() if st: description.append(st) cn = line[52:54].strip() if cn: description.append(cn) if description: stations[icao]["description"] = ", ".join(description) lat = line[55:60].strip() if lat: lat = int(lat)/100.0 lon = line[61:67].strip() if lon: lon = int(lon)/100.0 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) count += 1 metartbl.close() print("done (%s lines)." % count) message = "Reading %s..." % nsdcccc_fn sys.stdout.write(message) sys.stdout.flush() count = 0 nsdcccc = codecs.open(nsdcccc_fn, "rU", "latin1") for line in nsdcccc: line = str(line) fields = line.split(";") icao = fields[0].strip().lower() if icao in stations: description = [] name = " ".join( fields[3].strip().title().split() ) if name: description.append(name) st = fields[4].strip() if st: description.append(st) country = " ".join( fields[5].strip().title().split() ) if country: description.append(country) if description: stations[icao]["description"] = ", ".join(description) lat, lon = fields[7:9] if lat and lon: stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) elif "location" not in stations[icao]: lat, lon = fields[5:7] if lat and lon: stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) count += 1 nsdcccc.close() print("done (%s lines)." % count) message = "Reading %s..." % coopstn_fn sys.stdout.write(message) sys.stdout.flush() count = 0 coopstn = open(coopstn_fn) for line in coopstn: icao = line[33:37].strip().lower() if icao in stations: iata = line[22:26].strip().lower() if len(iata) == 3: airports[iata] = { "station": icao } if "description" not in stations[icao]: description = [] name = " ".join( line[99:129].strip().title().split() ) if name: description.append(name) st = line[59:61].strip() if st: description.append(st) country = " ".join( line[38:58].strip().title().split() ) if country: description.append(country) if description: stations[icao]["description"] = ", ".join(description) if "location" not in stations[icao]: lat = line[130:139].strip() if lat: lat = lat.replace(" ", "-") lon = line[140:150].strip() if lon: lon = lon.replace(" ", "-") stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) count += 1 coopstn.close() print("done (%s lines)." % count) message = "Reading %s..." % zlist_fn sys.stdout.write(message) sys.stdout.flush() count = 0 zlist = codecs.open(zlist_fn, "rU") for line in zlist: line = line.split("#")[0].strip() if line: zones[line] = {} count += 1 zlist.close() print("done (%s lines)." % count) message = "Reading %s:*..." % zcatalog_an sys.stdout.write(message) sys.stdout.flush() count = 0 zcatalog = tarfile.open(zcatalog_an) for entry in zcatalog.getmembers(): if entry.isfile(): fnmatch = re.match( r"([a-z]+z[0-9]+)\.txt$", os.path.basename(entry.name) ) if fnmatch: zone = fnmatch.group(1) if zone in zones: data = zcatalog.extractfile(entry).readlines() description = data[0].decode("ascii").strip() zones[zone]["description"] = description for line in data[1:]: line = line.decode("latin1").strip() urimatch = re.match("/webdocs/pub/(.+):(.+) for ", line) if urimatch: uritype = urimatch.group(2).lower().replace(" ","_") zones[zone][uritype] = ( "http://tgftp.nws.noaa.gov/%s" % urimatch.group(1)) count += 1 zcatalog.close() print("done (%s files)." % count) message = "Reading %s..." % cpfzcf_fn sys.stdout.write(message) sys.stdout.flush() count = 0 cpfz = {} cpfzcf = open(cpfzcf_fn) for line in cpfzcf: fields = line.strip().split("|") if len(fields) == 11 \ and fields[0] and fields[1] and fields[9] and fields[10]: zone = "z".join( fields[:2] ).lower() if zone in zones: zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) ) elif fields[6]: state = fields[0] description = fields[3] county = fields[5] fips = "fips%s"%fields[6] possible = [ "%s, %s" % (county, state), "%s County, %s" % (county, state), ] if description.endswith(" Counties"): description = description[:-9] for addition in description.split(" and "): possible.append( "%s, %s" % (addition, state) ) possible.append( "%s County, %s" % (addition, state) ) if fips in places and "centroid" in places[fips]: for candidate in zones: if "centroid" not in zones[candidate] and \ "description" in zones[candidate] and \ zones[candidate]["description"] in possible: zones[candidate]["centroid"] = \ places[fips]["centroid"] count += 1 cpfzcf.close() print("done (%s lines)." % count) message = "Reading %s:%s..." % (gzcta_an, gzcta_fn) sys.stdout.write(message) sys.stdout.flush() count = 0 gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU") columns = gzcta.readline().decode("latin1").strip().split("\t") for line in gzcta: fields = line.decode("latin1").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_intptlat = fields[ columns.index("INTPTLAT") ].strip() f_intptlong = fields[ columns.index("INTPTLONG") ].strip() if f_geoid and f_intptlat and f_intptlong: if f_geoid not in zctas: zctas[f_geoid] = {} zctas[f_geoid]["centroid"] = gecos( "%s,%s" % (f_intptlat, f_intptlong) ) count += 1 gzcta.close() print("done (%s lines)." % count) message = "Reading %s..." % overrides_fn sys.stdout.write(message) sys.stdout.flush() count = 0 added = 0 removed = 0 changed = 0 overrides = configparser.ConfigParser() overrides.readfp( codecs.open(overrides_fn, "r", "utf8") ) overrideslog = [] for section in overrides.sections(): addopt = 0 chgopt = 0 if section.startswith("-"): section = section[1:] delete = True else: delete = False if re.match("[A-Za-z]{3}$", section): if delete: if section in airports: del( airports[section] ) logact = "removed airport %s" % section removed += 1 else: logact = "tried to remove nonexistent airport %s" % section else: if section in airports: logact = "changed airport %s" % section changed += 1 else: airports[section] = {} logact = "added airport %s" % section added += 1 for key,value in overrides.items(section): if key in airports[section]: chgopt += 1 else: addopt += 1 if key in ("centroid", "location"): airports[section][key] = eval(value) else: airports[section][key] = value if addopt and chgopt: logact += " (+%s/!%s options)" % (addopt, chgopt) elif addopt: logact += " (+%s options)" % addopt elif chgopt: logact += " (!%s options)" % chgopt elif re.match("[A-Za-z0-9]{4}$", section): if delete: if section in stations: del( stations[section] ) logact = "removed station %s" % section removed += 1 else: logact = "tried to remove nonexistent station %s" % section else: if section in stations: logact = "changed station %s" % section changed += 1 else: stations[section] = {} logact = "added station %s" % section added += 1 for key,value in overrides.items(section): if key in stations[section]: chgopt += 1 else: addopt += 1 if key in ("centroid", "location"): stations[section][key] = eval(value) else: stations[section][key] = value if addopt and chgopt: logact += " (+%s/!%s options)" % (addopt, chgopt) elif addopt: logact += " (+%s options)" % addopt elif chgopt: logact += " (!%s options)" % chgopt elif re.match("[0-9]{5}$", section): if delete: if section in zctas: del( zctas[section] ) logact = "removed zcta %s" % section removed += 1 else: logact = "tried to remove nonexistent zcta %s" % section else: if section in zctas: logact = "changed zcta %s" % section changed += 1 else: zctas[section] = {} logact = "added zcta %s" % section added += 1 for key,value in overrides.items(section): if key in zctas[section]: chgopt += 1 else: addopt += 1 if key in ("centroid", "location"): zctas[section][key] = eval(value) else: zctas[section][key] = value if addopt and chgopt: logact += " (+%s/!%s options)" % (addopt, chgopt) elif addopt: logact += " (+%s options)" % addopt elif chgopt: logact += " (!%s options)" % chgopt elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section): if delete: if section in zones: del( zones[section] ) logact = "removed zone %s" % section removed += 1 else: logact = "tried to remove nonexistent zone %s" % section else: if section in zones: logact = "changed zone %s" % section changed += 1 else: zones[section] = {} logact = "added zone %s" % section added += 1 for key,value in overrides.items(section): if key in zones[section]: chgopt += 1 else: addopt += 1 if key in ("centroid", "location"): zones[section][key] = eval(value) else: zones[section][key] = value if addopt and chgopt: logact += " (+%s/!%s options)" % (addopt, chgopt) elif addopt: logact += " (+%s options)" % addopt elif chgopt: logact += " (!%s options)" % chgopt elif re.match("fips[0-9]+$", section): if delete: if section in places: del( places[section] ) logact = "removed place %s" % section removed += 1 else: logact = "tried to remove nonexistent place %s" % section else: if section in places: logact = "changed place %s" % section changed += 1 else: places[section] = {} logact = "added place %s" % section added += 1 for key,value in overrides.items(section): if key in places[section]: chgopt += 1 else: addopt += 1 if key in ("centroid", "location"): places[section][key] = eval(value) else: places[section][key] = value if addopt and chgopt: logact += " (+%s/!%s options)" % (addopt, chgopt) elif addopt: logact += " (+%s options)" % addopt elif chgopt: logact += " (!%s options)" % chgopt count += 1 overrideslog.append("%s\n" % logact) overrideslog.sort() if os.path.exists(overrideslog_fn): os.rename(overrideslog_fn, "%s_old"%overrideslog_fn) overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8") overrideslog_fd.writelines(overrideslog) overrideslog_fd.close() print("done (%s overridden sections: +%s/-%s/!%s)." % ( count, added, removed, changed ) ) estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones) print( "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \ estimate ) count = 0 milestones = list( range(51) ) message = " " sys.stdout.write(message) sys.stdout.flush() for fips in places: centroid = places[fips]["centroid"] if centroid: station = closest(centroid, stations, "location", 0.1) if station[0]: places[fips]["station"] = station count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[:milestones.index(level)+1]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) if centroid: zone = closest(centroid, zones, "centroid", 0.1) if zone[0]: places[fips]["zone"] = zone count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[:milestones.index(level)+1]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) for station in stations: if "location" in stations[station]: location = stations[station]["location"] if location: zone = closest(location, zones, "centroid", 0.1) if zone[0]: stations[station]["zone"] = zone count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[:milestones.index(level)+1]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) for zcta in zctas.keys(): centroid = zctas[zcta]["centroid"] if centroid: station = closest(centroid, stations, "location", 0.1) if station[0]: zctas[zcta]["station"] = station count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[ : milestones.index(level)+1 ]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) if centroid: zone = closest(centroid, zones, "centroid", 0.1) if zone[0]: zctas[zcta]["zone"] = zone count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[:milestones.index(level)+1]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) for zone in zones.keys(): if "centroid" in zones[zone]: centroid = zones[zone]["centroid"] if centroid: station = closest(centroid, stations, "location", 0.1) if station[0]: zones[zone]["station"] = station count += 1 if not count%100: level = int(50*count/estimate) if level in milestones: for remaining in milestones[:milestones.index(level)+1]: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() milestones.remove(remaining) for remaining in milestones: if remaining%5: message = "." sys.stdout.write(message) sys.stdout.flush() else: message = "%s%%" % (remaining*2,) sys.stdout.write(message) sys.stdout.flush() print("\n done (%s correlations)." % count) message = "Writing %s..." % airports_fn sys.stdout.write(message) sys.stdout.flush() count = 0 if os.path.exists(airports_fn): os.rename(airports_fn, "%s_old"%airports_fn) airports_fd = codecs.open(airports_fn, "w", "utf8") airports_fd.write(header) for airport in sorted( airports.keys() ): airports_fd.write("\n\n[%s]" % airport) for key, value in sorted( airports[airport].items() ): if type(value) is float: value = "%.7f"%value elif type(value) is tuple: elements = [] for element in value: if type(element) is float: elements.append("%.7f"%element) else: elements.append( repr(element) ) value = "(%s)"%", ".join(elements) airports_fd.write( "\n%s = %s" % (key, value) ) count += 1 airports_fd.write("\n") airports_fd.close() print("done (%s sections)." % count) message = "Writing %s..." % places_fn sys.stdout.write(message) sys.stdout.flush() count = 0 if os.path.exists(places_fn): os.rename(places_fn, "%s_old"%places_fn) places_fd = codecs.open(places_fn, "w", "utf8") places_fd.write(header) for fips in sorted( places.keys() ): places_fd.write("\n\n[%s]" % fips) for key, value in sorted( places[fips].items() ): if type(value) is float: value = "%.7f"%value elif type(value) is tuple: elements = [] for element in value: if type(element) is float: elements.append("%.7f"%element) else: elements.append( repr(element) ) value = "(%s)"%", ".join(elements) places_fd.write( "\n%s = %s" % (key, value) ) count += 1 places_fd.write("\n") places_fd.close() print("done (%s sections)." % count) message = "Writing %s..." % stations_fn sys.stdout.write(message) sys.stdout.flush() count = 0 if os.path.exists(stations_fn): os.rename(stations_fn, "%s_old"%stations_fn) stations_fd = codecs.open(stations_fn, "w", "utf8") stations_fd.write(header) for station in sorted( stations.keys() ): stations_fd.write("\n\n[%s]" % station) for key, value in sorted( stations[station].items() ): if type(value) is float: value = "%.7f"%value elif type(value) is tuple: elements = [] for element in value: if type(element) is float: elements.append("%.7f"%element) else: elements.append( repr(element) ) value = "(%s)"%", ".join(elements) stations_fd.write( "\n%s = %s" % (key, value) ) count += 1 stations_fd.write("\n") stations_fd.close() print("done (%s sections)." % count) message = "Writing %s..." % zctas_fn sys.stdout.write(message) sys.stdout.flush() count = 0 if os.path.exists(zctas_fn): os.rename(zctas_fn, "%s_old"%zctas_fn) zctas_fd = codecs.open(zctas_fn, "w", "utf8") zctas_fd.write(header) for zcta in sorted( zctas.keys() ): zctas_fd.write("\n\n[%s]" % zcta) for key, value in sorted( zctas[zcta].items() ): if type(value) is float: value = "%.7f"%value elif type(value) is tuple: elements = [] for element in value: if type(element) is float: elements.append("%.7f"%element) else: elements.append( repr(element) ) value = "(%s)"%", ".join(elements) zctas_fd.write( "\n%s = %s" % (key, value) ) count += 1 zctas_fd.write("\n") zctas_fd.close() print("done (%s sections)." % count) message = "Writing %s..." % zones_fn sys.stdout.write(message) sys.stdout.flush() count = 0 if os.path.exists(zones_fn): os.rename(zones_fn, "%s_old"%zones_fn) zones_fd = codecs.open(zones_fn, "w", "utf8") zones_fd.write(header) for zone in sorted( zones.keys() ): zones_fd.write("\n\n[%s]" % zone) for key, value in sorted( zones[zone].items() ): if type(value) is float: value = "%.7f"%value elif type(value) is tuple: elements = [] for element in value: if type(element) is float: elements.append("%.7f"%element) else: elements.append( repr(element) ) value = "(%s)"%", ".join(elements) zones_fd.write( "\n%s = %s" % (key, value) ) count += 1 zones_fd.write("\n") zones_fd.close() print("done (%s sections)." % count) message = "Starting QA check..." sys.stdout.write(message) sys.stdout.flush() airports = configparser.ConfigParser() airports.read(airports_fn) places = configparser.ConfigParser() places.read(places_fn) stations = configparser.ConfigParser() stations.read(stations_fn) zctas = configparser.ConfigParser() zctas.read(zctas_fn) zones = configparser.ConfigParser() zones.read(zones_fn) qalog = [] places_nocentroid = 0 places_nodescription = 0 for place in sorted( places.sections() ): if not places.has_option(place, "centroid"): qalog.append("%s: no centroid\n" % place) places_nocentroid += 1 if not places.has_option(place, "description"): qalog.append("%s: no description\n" % place) places_nodescription += 1 stations_nodescription = 0 stations_nolocation = 0 stations_nometar = 0 for station in sorted( stations.sections() ): if not stations.has_option(station, "description"): qalog.append("%s: no description\n" % station) stations_nodescription += 1 if not stations.has_option(station, "location"): qalog.append("%s: no location\n" % station) stations_nolocation += 1 if not stations.has_option(station, "metar"): qalog.append("%s: no metar\n" % station) stations_nometar += 1 airports_badstation = 0 airports_nostation = 0 for airport in sorted( airports.sections() ): if not airports.has_option(airport, "station"): qalog.append("%s: no station\n" % airport) airports_nostation += 1 else: station = airports.get(airport, "station") if station not in stations.sections(): qalog.append( "%s: bad station %s\n" % (airport, station) ) airports_badstation += 1 zctas_nocentroid = 0 for zcta in sorted( zctas.sections() ): if not zctas.has_option(zcta, "centroid"): qalog.append("%s: no centroid\n" % zcta) zctas_nocentroid += 1 zones_nocentroid = 0 zones_nodescription = 0 zones_noforecast = 0 zones_overlapping = 0 zonetable = {} for zone in zones.sections(): if zones.has_option(zone, "centroid"): zonetable[zone] = { "centroid": eval( zones.get(zone, "centroid") ) } for zone in sorted( zones.sections() ): if zones.has_option(zone, "centroid"): zonetable_local = zonetable.copy() del( zonetable_local[zone] ) centroid = eval( zones.get(zone, "centroid") ) if centroid: nearest = closest(centroid, zonetable_local, "centroid", 0.1) if nearest[1]*radian_to_km < 1: qalog.append( "%s: within one km of %s\n" % ( zone, nearest[0] ) ) zones_overlapping += 1 else: qalog.append("%s: no centroid\n" % zone) zones_nocentroid += 1 if not zones.has_option(zone, "description"): qalog.append("%s: no description\n" % zone) zones_nodescription += 1 if not zones.has_option(zone, "zone_forecast"): qalog.append("%s: no forecast\n" % zone) zones_noforecast += 1 if os.path.exists(qalog_fn): os.rename(qalog_fn, "%s_old"%qalog_fn) qalog_fd = codecs.open(qalog_fn, "w", "utf8") qalog_fd.writelines(qalog) qalog_fd.close() if qalog: print("issues found (see %s for details):"%qalog_fn) if airports_badstation: print(" %s airports with invalid station"%airports_badstation) if airports_nostation: print(" %s airports with no station"%airports_nostation) if places_nocentroid: print(" %s places with no centroid"%places_nocentroid) if places_nodescription: print(" %s places with no description"%places_nodescription) if stations_nodescription: print(" %s stations with no description"%stations_nodescription) if stations_nolocation: print(" %s stations with no location"%stations_nolocation) if stations_nometar: print(" %s stations with no METAR"%stations_nometar) if zctas_nocentroid: print(" %s ZCTAs with no centroid"%zctas_nocentroid) if zones_nocentroid: print(" %s zones with no centroid"%zones_nocentroid) if zones_nodescription: print(" %s zones with no description"%zones_nodescription) if zones_noforecast: print(" %s zones with no forecast"%zones_noforecast) if zones_overlapping: print(" %s zones within one km of another"%zones_overlapping) else: print("no issues found.") print("Indexing complete!")