From e1070965f262610bf9aa5210a01a1c74e4b7521b Mon Sep 17 00:00:00 2001 From: sparky8512 <76499194+sparky8512@users.noreply.github.com> Date: Tue, 7 Sep 2021 17:29:56 -0700 Subject: [PATCH 1/2] Initial cut of obstruction map support Add a new command line script, dish_obstruction_map.py, that writes a PNG image based on the obstruction map data queried from the dish. Supports color or greyscale output and either with or without alpha channel. Does not yet support running in an interval loop, mostly because that will require templatizing the output filename in order to be useful. Tracked on issue #27 --- dish_obstruction_map.py | 150 ++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + starlink_grpc.py | 47 +++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 dish_obstruction_map.py diff --git a/dish_obstruction_map.py b/dish_obstruction_map.py new file mode 100644 index 0000000..1ecf63d --- /dev/null +++ b/dish_obstruction_map.py @@ -0,0 +1,150 @@ +#!/usr/bin/python3 +"""Write a PNG image representing Starlink obstruction map data. + +This scripts queries obstruction map data from the Starlink user terminal +reachable on the local network and writes a PNG image based on that data. +""" + +import argparse +import logging +import os +import png +import sys + +import starlink_grpc + +DEFAULT_OBSTRUCTED_COLOR = "FFFF0000" +DEFAULT_UNOBSTRUCTED_COLOR = "FFFFFFFF" +DEFAULT_NO_DATA_COLOR = "00000000" +DEFAULT_OBSTRUCTED_GREYSCALE = "FF00" +DEFAULT_UNOBSTRUCTED_GREYSCALE = "FFFF" +DEFAULT_NO_DATA_GREYSCALE = "0000" + + +def run_loop(opts, context): + snr_data = starlink_grpc.obstruction_map(context) + + def pixel_bytes(row): + for point in row: + if point > 1.0: + # shouldn't happen, but just in case... + point = 1.0 + + if point >= 0.0: + if opts.greyscale: + yield round(point * opts.unobstructed_color_g + + (1.0-point) * opts.obstructed_color_g) + else: + yield round(point * opts.unobstructed_color_r + + (1.0-point) * opts.obstructed_color_r) + yield round(point * opts.unobstructed_color_g + + (1.0-point) * opts.obstructed_color_g) + yield round(point * opts.unobstructed_color_b + + (1.0-point) * opts.obstructed_color_b) + if not opts.no_alpha: + yield round(point * opts.unobstructed_color_a + + (1.0-point) * opts.obstructed_color_a) + else: + if opts.greyscale: + yield opts.no_data_color_g + else: + yield opts.no_data_color_r + yield opts.no_data_color_g + yield opts.no_data_color_b + if not opts.no_alpha: + yield opts.no_data_color_a + + if opts.filename == "-": + # Open new stdout file to get binary mode + out_file = os.fdopen(sys.stdout.fileno(), "wb", closefd=False) + else: + out_file = open(opts.filename, "wb") + if not snr_data or not snr_data[0]: + logging.error("Invalid SNR map data: Zero-length") + return 1 + writer = png.Writer(len(snr_data[0]), + len(snr_data), + alpha=(not opts.no_alpha), + greyscale=opts.greyscale) + writer.write(out_file, (bytes(pixel_bytes(row)) for row in snr_data)) + out_file.close() + return 0 + + +def main(): + logging.basicConfig(format="%(levelname)s: %(message)s") + + parser = argparse.ArgumentParser( + description="Collect directional obstruction map data from a Starlink user terminal and " + "emit it as a PNG image") + parser.add_argument("filename", help="The image file to write, or - to write to stdout") + parser.add_argument( + "-o", + "--obstructed-color", + help="Color of obstructed areas, in RGB, ARGB, L, or AL hex notation, default: " + + DEFAULT_OBSTRUCTED_COLOR + " or " + DEFAULT_OBSTRUCTED_GREYSCALE) + parser.add_argument( + "-u", + "--unobstructed-color", + help="Color of unobstructed areas, in RGB, ARGB, L, or AL hex notation, default: " + + DEFAULT_UNOBSTRUCTED_COLOR + " or " + DEFAULT_UNOBSTRUCTED_GREYSCALE) + parser.add_argument( + "-n", + "--no-data-color", + help="Color of areas with no data, in RGB, ARGB, L, or AL hex notation, default: " + + DEFAULT_NO_DATA_COLOR + " or " + DEFAULT_NO_DATA_GREYSCALE) + parser.add_argument( + "-g", + "--greyscale", + action="store_true", + help= + "Emit a greyscale image instead of the default full color image; greyscale images use L or AL hex notation for the color options" + ) + parser.add_argument( + "-z", + "--no-alpha", + action="store_true", + help= + "Emit an image without alpha (transparency) channel instead of the default that includes alpha channel" + ) + parser.add_argument("-t", + "--target", + help="host:port of dish to query, default is the standard IP address " + "and port (192.168.100.1:9200)") + opts = parser.parse_args() + + if opts.obstructed_color is None: + opts.obstructed_color = DEFAULT_OBSTRUCTED_GREYSCALE if opts.greyscale else DEFAULT_OBSTRUCTED_COLOR + if opts.unobstructed_color is None: + opts.unobstructed_color = DEFAULT_UNOBSTRUCTED_GREYSCALE if opts.greyscale else DEFAULT_UNOBSTRUCTED_COLOR + if opts.no_data_color is None: + opts.no_data_color = DEFAULT_NO_DATA_GREYSCALE if opts.greyscale else DEFAULT_NO_DATA_COLOR + + for option in ("obstructed_color", "unobstructed_color", "no_data_color"): + try: + color = int(getattr(opts, option), 16) + if opts.greyscale: + setattr(opts, option + "_a", (color >> 8) & 255) + setattr(opts, option + "_g", color & 255) + else: + setattr(opts, option + "_a", (color >> 24) & 255) + setattr(opts, option + "_r", (color >> 16) & 255) + setattr(opts, option + "_g", (color >> 8) & 255) + setattr(opts, option + "_b", color & 255) + except ValueError: + logging.error("Invalid hex number for %s", option) + sys.exit(1) + + context = starlink_grpc.ChannelContext(target=opts.target) + + try: + # XXX: make this actually run in a loop... + rc = run_loop(opts, context) + finally: + context.close() + + sys.exit(rc) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 8dae5fc..55716c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ protobuf>=3.6.0 yagrc>=1.1.0 paho-mqtt>=1.5.1 influxdb>=5.3.1 +pypng>=0.0.20 diff --git a/starlink_grpc.py b/starlink_grpc.py index 7e37649..56491d9 100644 --- a/starlink_grpc.py +++ b/starlink_grpc.py @@ -1182,3 +1182,50 @@ def history_stats(parse_samples, start=None, verbose=False, context=None, histor "download_usage": int(round(usage_down / 8)), "upload_usage": int(round(usage_up / 8)), } + + +def get_obstruction_map(context=None): + """Fetch obstruction map data and return it in grpc structure format. + + Args: + context (ChannelContext): Optionally provide a channel for reuse + across repeated calls. If an existing channel is reused, the RPC + call will be retried at most once, since connectivity may have + been lost and restored in the time since it was last used. + + Raises: + grpc.RpcError: Communication or service error. + """ + def grpc_call(channel): + if imports_pending: + resolve_imports(channel) + stub = device_pb2_grpc.DeviceStub(channel) + response = stub.Handle(device_pb2.Request(dish_get_obstruction_map={})) + return response.dish_get_obstruction_map + + return call_with_channel(grpc_call, context=context) + + +def obstruction_map(context=None): + """Fetch current obstruction map data. + + Args: + context (ChannelContext): Optionally provide a channel for reuse + across repeated calls. + + Returns: + A tuple of row data, each of which is a tuple of column data, which + hold floats indicating SNR info per direction in the range of 0.0 to + 1.0 for valid data and -1.0 for invalid data. To get a flat + representation the SNR data instead, see `get_obstruction_map`. + + Raises: + GrpcError: Failed getting status info from the Starlink user terminal. + """ + try: + map_data = get_obstruction_map(context) + except grpc.RpcError as e: + raise GrpcError(e) + + cols = map_data.num_cols + return tuple((map_data.snr[i:i + cols]) for i in range(0, cols * map_data.num_rows, cols)) From 1a9af6ad5d7605cd3f2a224d574bcc43c923eecd Mon Sep 17 00:00:00 2001 From: sparky8512 <76499194+sparky8512@users.noreply.github.com> Date: Wed, 8 Sep 2021 13:45:36 -0700 Subject: [PATCH 2/2] Interval loop support for obstruction maps Tracked on issue #27 --- README.md | 17 ++++++++++- dish_obstruction_map.py | 66 +++++++++++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3b350b5..3a99ce7 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,13 @@ The scripts that use [MQTT](https://mqtt.org/) for output require the `paho-mqtt The scripts that use [InfluxDB](https://www.influxdata.com/products/influxdb/) for output require the `influxdb` Python package. Information about how to install that can be found at https://github.com/influxdata/influxdb-python. Note that this is the (slightly) older version of the InfluxDB client Python module, not the InfluxDB 2.0 client. It can still be made to work with an InfluxDB 2.0 server, but doing so requires using `influx v1` [CLI commands](https://docs.influxdata.com/influxdb/v2.0/reference/cli/influx/v1/) on the server to map the 1.x username, password, and database names to their 2.0 equivalents. +The `dish_obstruction_map.py` script requires the `pypng` Python package. Information about how to install that can be found at https://pypng.readthedocs.io/en/latest/png.html#installation-and-overview + Note that the Python package versions available from various Linux distributions (ie: installed via `apt-get` or similar) tend to run a bit behind those available to install via `pip`. While the distro packages should work OK as long as they aren't extremely old, they may not work as well as the later versions. ### Generating the gRPC protocol modules -This step is no longer required, as the grpc scripts can now get the protocol module classes at run time via reflection, but generating the protocol modules will improve script startup time, and it would be a good idea to at least stash away the protoset file emitted by `grpcurl` in case SpaceX ever turns off server reflection in the dish software. +This step is no longer required, as the grpc scripts can now get the protocol module classes at run time via reflection, but generating the protocol modules will improve script startup time, and it would be a good idea to at least stash away the protoset file emitted by `grpcurl` in case SpaceX ever turns off server reflection in the dish software. That being said, it's probably less error prone to use the run time reflection support rather than using the files generated in this step, so use your own judgement as to whether or not to proceed with this. The grpc scripts require some generated code to support the specific gRPC protocol messages used. These would normally be generated from .proto files that specify those messages, but to date (2020-Dec), SpaceX has not publicly released such files. The gRPC service running on the dish appears to have [server reflection](https://github.com/grpc/grpc/blob/master/doc/server-reflection.md) enabled, though. `grpcurl` can use that to extract a protoset file, and the `protoc` compiler can use that to make the necessary generated code: ```shell script @@ -86,6 +88,19 @@ python3 dish_grpc_text.py -t 60 -o 60 ping_drop ``` will poll history data once per minute, but compute statistics only once per hour. This also reduces data loss due to a dish reboot, since the `-o` option will aggregate across reboots, too. +### The obstruction map script + +`dish_obstruction_map.py` is a little different in that it doesn't write to a database, but rather writes PNG images to the local filesystem. To get a single image of the current obstruction map using the default colors, you can do the following: +```shell script +python3 dish_obstruction_map.py obstructions.png +``` +or to run in a loop writing a sequence of images once per hour, you can do the following: +```shell script +python3 dish_obstruction_map.py -t 3600 obstructions_%s.png +``` + +Run it with the `-h` command line option for full usage details, including control of the map colors and color modes. + ### The JSON parser script `dish_json_text.py` is similar to `dish_grpc_text.py`, but it takes JSON format input from a file instead of pulling it directly from the dish via grpc call. It also does not support the status info modes, because those are easy enough to interpret directly from the JSON data. The easiest way to use it is to pipe the `grpcurl` command directly into it. For example: diff --git a/dish_obstruction_map.py b/dish_obstruction_map.py index 1ecf63d..636551f 100644 --- a/dish_obstruction_map.py +++ b/dish_obstruction_map.py @@ -6,10 +6,12 @@ reachable on the local network and writes a PNG image based on that data. """ import argparse +from datetime import datetime import logging import os import png import sys +import time import starlink_grpc @@ -19,9 +21,10 @@ DEFAULT_NO_DATA_COLOR = "00000000" DEFAULT_OBSTRUCTED_GREYSCALE = "FF00" DEFAULT_UNOBSTRUCTED_GREYSCALE = "FFFF" DEFAULT_NO_DATA_GREYSCALE = "0000" +LOOP_TIME_DEFAULT = 0 -def run_loop(opts, context): +def loop_body(opts, context): snr_data = starlink_grpc.obstruction_map(context) def pixel_bytes(row): @@ -58,7 +61,12 @@ def run_loop(opts, context): # Open new stdout file to get binary mode out_file = os.fdopen(sys.stdout.fileno(), "wb", closefd=False) else: - out_file = open(opts.filename, "wb") + now = int(time.time()) + filename = opts.filename.replace("%u", str(now)) + filename = filename.replace("%d", + datetime.utcfromtimestamp(now).strftime("%Y_%m_%d_%H_%M_%S")) + filename = filename.replace("%s", str(opts.sequence)) + out_file = open(filename, "wb") if not snr_data or not snr_data[0]: logging.error("Invalid SNR map data: Zero-length") return 1 @@ -68,16 +76,20 @@ def run_loop(opts, context): greyscale=opts.greyscale) writer.write(out_file, (bytes(pixel_bytes(row)) for row in snr_data)) out_file.close() + + opts.sequence += 1 return 0 -def main(): - logging.basicConfig(format="%(levelname)s: %(message)s") - +def parse_args(): parser = argparse.ArgumentParser( description="Collect directional obstruction map data from a Starlink user terminal and " "emit it as a PNG image") - parser.add_argument("filename", help="The image file to write, or - to write to stdout") + parser.add_argument( + "filename", + help="The image file to write, or - to write to stdout; may be a template with the " + "following to be filled in per loop iteration: %%s for sequence number, %%d for UTC date " + "and time, %%u for seconds since Unix epoch.") parser.add_argument( "-o", "--obstructed-color", @@ -97,20 +109,29 @@ def main(): "-g", "--greyscale", action="store_true", - help= - "Emit a greyscale image instead of the default full color image; greyscale images use L or AL hex notation for the color options" - ) + help="Emit a greyscale image instead of the default full color image; greyscale images " + "use L or AL hex notation for the color options") parser.add_argument( "-z", "--no-alpha", action="store_true", - help= - "Emit an image without alpha (transparency) channel instead of the default that includes alpha channel" - ) - parser.add_argument("-t", + help="Emit an image without alpha (transparency) channel instead of the default that " + "includes alpha channel") + parser.add_argument("-e", "--target", help="host:port of dish to query, default is the standard IP address " "and port (192.168.100.1:9200)") + parser.add_argument("-t", + "--loop-interval", + type=float, + default=float(LOOP_TIME_DEFAULT), + help="Loop interval in seconds or 0 for no loop, default: " + + str(LOOP_TIME_DEFAULT)) + parser.add_argument("-s", + "--sequence", + type=int, + default=1, + help="Starting sequence number for templatized filenames, default: 1") opts = parser.parse_args() if opts.obstructed_color is None: @@ -135,11 +156,26 @@ def main(): logging.error("Invalid hex number for %s", option) sys.exit(1) + return opts + + +def main(): + opts = parse_args() + + logging.basicConfig(format="%(levelname)s: %(message)s") + context = starlink_grpc.ChannelContext(target=opts.target) try: - # XXX: make this actually run in a loop... - rc = run_loop(opts, context) + next_loop = time.monotonic() + while True: + rc = loop_body(opts, context) + if opts.loop_interval > 0.0: + now = time.monotonic() + next_loop = max(next_loop + opts.loop_interval, now) + time.sleep(next_loop - now) + else: + break finally: context.close()