From e1a4c473c8a5541f9f2f48f0789f6630f21194fa Mon Sep 17 00:00:00 2001 From: sparky8512 <76499194+sparky8512@users.noreply.github.com> Date: Wed, 30 Dec 2020 10:17:02 -0800 Subject: [PATCH] Handle errors on the gRPC connection Also, actually do the thing I said I was doing in the prior checkin by writing state as a string instead of integer. And a bit more cleanup. --- dishDumpStatus.py | 2 + dishStatusInflux.py | 89 +++++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/dishDumpStatus.py b/dishDumpStatus.py index befa1e5..118d978 100644 --- a/dishDumpStatus.py +++ b/dishDumpStatus.py @@ -10,6 +10,8 @@ import grpc import spacex.api.device.device_pb2 import spacex.api.device.device_pb2_grpc +# Note that if you remove the 'with' clause here, you need to separately +# call channel.close() when you're done with the gRPC connection. with grpc.insecure_channel('192.168.100.1:9200') as channel: stub = spacex.api.device.device_pb2_grpc.DeviceStub(channel) response = stub.Handle(spacex.api.device.device_pb2.Request(get_status={})) diff --git a/dishStatusInflux.py b/dishStatusInflux.py index e39150c..e8658bb 100644 --- a/dishStatusInflux.py +++ b/dishStatusInflux.py @@ -17,6 +17,9 @@ import spacex.api.device.device_pb2_grpc import time +fVerbose = True +sleepTime = 30 + class DeviceStatusSeries(SeriesHelper): class Meta: series_name = "spacex.starlink.user_terminal.status" @@ -38,46 +41,78 @@ class DeviceStatusSeries(SeriesHelper): "fraction_obstructed"] tags = ["id"] -influxClient = InfluxDBClient("localhost", 8086, "script-user", "password", "dishstats", ssl=False, retries=1, timeout=15) +influxClient = InfluxDBClient(host="localhost", port=8086, username="script-user", password="password", database="dishstats", ssl=False, retries=1, timeout=15) try: - dishChannel = grpc.insecure_channel("192.168.100.1:9200") + dishChannel = None + lastId = None + fLastFailed = False pending = 0 count = 0 while True: - stub = spacex.api.device.device_pb2_grpc.DeviceStub(dishChannel) - response = stub.Handle(spacex.api.device.device_pb2.Request(get_status={})) - status = response.dish_get_status - DeviceStatusSeries( - id=status.device_info.id, - hardware_version=status.device_info.hardware_version, - software_version=status.device_info.software_version, - state=status.state, - alert_motors_stuck=status.alerts.motors_stuck, - alert_thermal_throttle=status.alerts.thermal_throttle, - alert_thermal_shutdown=status.alerts.thermal_shutdown, - alert_unexpected_location=status.alerts.unexpected_location, - snr=status.snr, - seconds_to_first_nonempty_slot=status.seconds_to_first_nonempty_slot, - pop_ping_drop_rate=status.pop_ping_drop_rate, - downlink_throughput_bps=status.downlink_throughput_bps, - uplink_throughput_bps=status.uplink_throughput_bps, - pop_ping_latency_ms=status.pop_ping_latency_ms, - currently_obstructed=status.obstruction_stats.currently_obstructed, - fraction_obstructed=status.obstruction_stats.fraction_obstructed) + try: + if dishChannel is None: + dishChannel = grpc.insecure_channel("192.168.100.1:9200") + stub = spacex.api.device.device_pb2_grpc.DeviceStub(dishChannel) + response = stub.Handle(spacex.api.device.device_pb2.Request(get_status={})) + status = response.dish_get_status + DeviceStatusSeries( + id=status.device_info.id, + hardware_version=status.device_info.hardware_version, + software_version=status.device_info.software_version, + state=spacex.api.device.dish_pb2.DishState.Name(status.state), + alert_motors_stuck=status.alerts.motors_stuck, + alert_thermal_throttle=status.alerts.thermal_throttle, + alert_thermal_shutdown=status.alerts.thermal_shutdown, + alert_unexpected_location=status.alerts.unexpected_location, + snr=status.snr, + seconds_to_first_nonempty_slot=status.seconds_to_first_nonempty_slot, + pop_ping_drop_rate=status.pop_ping_drop_rate, + downlink_throughput_bps=status.downlink_throughput_bps, + uplink_throughput_bps=status.uplink_throughput_bps, + pop_ping_latency_ms=status.pop_ping_latency_ms, + currently_obstructed=status.obstruction_stats.currently_obstructed, + fraction_obstructed=status.obstruction_stats.fraction_obstructed) + lastId = status.device_info.id + fLastFailed = False + except Exception as e: + if not dishChannel is None: + dishChannel.close() + dishChannel = None + if fLastFailed: + if not lastId is None: + DeviceStatusSeries(id=lastId, state="DISH_UNREACHABLE") + else: + # Retry once, because the connection may have been lost while + # we were sleeping + fLastFailed = True + continue pending = pending + 1 - print("Samples: " + str(pending)) + if fVerbose: + print("Samples: " + str(pending)) + count = count + 1 if count > 5: try: DeviceStatusSeries.commit(influxClient) - print("Wrote " + str(pending)) + if fVerbose: + print("Wrote " + str(pending)) pending = 0 except Exception as e: print("Failed to write: " + str(e)) count = 0 - count = count + 1 - time.sleep(5) + if sleepTime > 0: + time.sleep(sleepTime) + else: + break finally: # Flush on error/exit - DeviceStatusSeries.commit(influxClient) + try: + DeviceStatusSeries.commit(influxClient) + if fVerbose: + print("Wrote " + str(pending)) + except Exception as e: + print("Failed to write: " + str(e)) + influxClient.close() + if not dishChannel is None: + dishChannel.close()