From f0aa1fe8dc55678e3df06971a2371432f3f48a3d Mon Sep 17 00:00:00 2001 From: sparky8512 <76499194+sparky8512@users.noreply.github.com> Date: Thu, 24 Dec 2020 14:58:28 -0800 Subject: [PATCH] Add (optional) run length stats Add reporting of ping drop runs per length of run. Useful(?) for characterizing how long a typical "outage" is. --- dishHistoryStats.py | 52 ++++++++++++++++++++++++++++++++++++++++++--- parseJsonHistory.py | 51 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/dishHistoryStats.py b/dishHistoryStats.py index c700dbf..5dd1a51 100644 --- a/dishHistoryStats.py +++ b/dishHistoryStats.py @@ -25,7 +25,7 @@ from itertools import chain fArgError = False try: - opts, args = getopt.getopt(sys.argv[1:], "ahs:vH") + opts, args = getopt.getopt(sys.argv[1:], "ahrs:vH") except getopt.GetoptError as err: print(str(err)) fArgError = True @@ -36,6 +36,7 @@ fUsage = False fVerbose = False fParseAll = False fHeader = False +fRunLengths = False if not fArgError: if len(args) > 0: @@ -46,6 +47,8 @@ if not fArgError: fParseAll = True elif opt == "-h": fUsage = True + elif opt == "-r": + fRunLengths = True elif opt == "-s": parseSamples = int(arg) elif opt == "-v": @@ -58,18 +61,28 @@ if fUsage or fArgError: print("Options:") print(" -a: Parse all valid samples") print(" -h: Be helpful") + print(" -r: Include ping drop run length stats") print(" -s : Parse data samples, default: "+str(parseSamples)) print(" -v: Be verbose") print(" -H: print CSV header instead of parsing file") sys.exit(1 if fArgError else 0) if fHeader: - print("datetimestamp_utc,samples,total_ping_drop,count_full_ping_drop,count_obstructed,total_obstructed_ping_drop,count_full_obstructed_ping_drop,count_unscheduled,total_unscheduled_ping_drop,count_full_unscheduled_ping_drop") + header = "datetimestamp_utc,samples,total_ping_drop,count_full_ping_drop,count_obstructed,total_obstructed_ping_drop,count_full_obstructed_ping_drop,count_unscheduled,total_unscheduled_ping_drop,count_full_unscheduled_ping_drop" + if fRunLengths: + header+= ",init_run_fragment,final_run_fragment," + header += ",".join("run_seconds_" + str(x) for x in range(1, 61)) + "," + header += ",".join("run_minutes_" + str(x) for x in range(1, 60)) + header += ",run_minutes_60_or_greater" + print(header) sys.exit(0) with grpc.insecure_channel('192.168.100.1:9200') as channel: stub = spacex.api.device.device_pb2_grpc.DeviceStub(channel) response = stub.Handle(spacex.api.device.device_pb2.Request(get_history={})) + +timestamp = datetime.datetime.utcnow() + historyData = response.dish_get_history # 'current' is the count of data samples written to the ring buffer, @@ -99,6 +112,11 @@ totObstruct = 0 totObstructD = 0 totObstructOne = 0 +secondRuns = [0] * 60 +minuteRuns = [0] * 60 +runLength = 0 +initRun = None + if fParseAll or nSamples < parseSamples: parseSamples = nSamples @@ -115,6 +133,18 @@ for i in sampleRange: tot += d if d >= 1: totOne += d + runLength += 1 + elif runLength > 0: + if initRun is None: + initRun = runLength + else: + if runLength <= 60: + secondRuns[runLength-1] += runLength + else: + minuteRuns[min((runLength-1)//60-1, 59)] += runLength + runLength = 0 + elif initRun is None: + initRun = 0 if not historyData.scheduled[i]: totUnsched += 1 totUnschedD += d @@ -126,6 +156,14 @@ for i in sampleRange: if d >= 1: totObstructOne += d +# If the entire sample set is one big drop run, it will be both initial +# fragment (continued from prior sample range) and final one (continued +# to next sample range), but to avoid double-reporting, just call it +# the initial run. +if initRun is None: + initRun = runLength + runLength = 0 + if fVerbose: print("Parsed samples: " + str(parseSamples)) print("Total ping drop: " + str(tot)) @@ -136,6 +174,14 @@ if fVerbose: print("Unscheduled: " + str(totUnsched)) print("Unscheduled ping drop: " + str(totUnschedD)) print("Unscheduled drop == 1: " + str(totUnschedOne)) + if fRunLengths: + print("Initial drop run fragment: " + str(initRun)) + print("Final drop run fragment: " + str(runLength)) + print("Per-second drop runs: " + ", ".join(str(x) for x in secondRuns)) + print("Per-minute drop runs: " + ", ".join(str(x) for x in minuteRuns)) else: # NOTE: When changing data output format, also change the -H header printing above. - print(datetime.datetime.utcnow().replace(microsecond=0).isoformat()+","+str(parseSamples)+","+str(tot)+","+str(totOne)+","+str(totObstruct)+","+str(totObstructD)+","+str(totObstructOne)+","+str(totUnsched)+","+str(totUnschedD)+","+str(totUnschedOne)) + csvData = timestamp.replace(microsecond=0).isoformat() + "," + ",".join(str(x) for x in [parseSamples, tot, totOne, totObstruct, totObstructD, totObstructOne, totUnsched, totUnschedD, totUnschedOne]) + if fRunLengths: + csvData += "," + ",".join(str(x) for x in chain([initRun, runLength], secondRuns, minuteRuns)) + print(csvData) diff --git a/parseJsonHistory.py b/parseJsonHistory.py index 69b182b..8c80386 100644 --- a/parseJsonHistory.py +++ b/parseJsonHistory.py @@ -23,7 +23,7 @@ from itertools import chain fArgError = False try: - opts, args = getopt.getopt(sys.argv[1:], "ahs:vH") + opts, args = getopt.getopt(sys.argv[1:], "ahrs:vH") except getopt.GetoptError as err: print(str(err)) fArgError = True @@ -34,6 +34,7 @@ fUsage = False fVerbose = False fParseAll = False fHeader = False +fRunLengths = False if not fArgError: if len(args) > 1: @@ -44,6 +45,8 @@ if not fArgError: fParseAll = True elif opt == "-h": fUsage = True + elif opt == "-r": + fRunLengths = True elif opt == "-s": parseSamples = int(arg) elif opt == "-v": @@ -57,13 +60,20 @@ if fUsage or fArgError: print("Options:") print(" -a: Parse all valid samples") print(" -h: Be helpful") + print(" -r: Include ping drop run length stats") print(" -s : Parse data samples, default: "+str(parseSamples)) print(" -v: Be verbose") print(" -H: print CSV header instead of parsing file") sys.exit(1 if fArgError else 0) if fHeader: - print("datetimestamp_utc,samples,total_ping_drop,count_full_ping_drop,count_obstructed,total_obstructed_ping_drop,count_full_obstructed_ping_drop,count_unscheduled,total_unscheduled_ping_drop,count_full_unscheduled_ping_drop") + header = "datetimestamp_utc,samples,total_ping_drop,count_full_ping_drop,count_obstructed,total_obstructed_ping_drop,count_full_obstructed_ping_drop,count_unscheduled,total_unscheduled_ping_drop,count_full_unscheduled_ping_drop" + if fRunLengths: + header+= ",init_run_fragment,final_run_fragment," + header += ",".join("run_seconds_" + str(x) for x in range(1, 61)) + "," + header += ",".join("run_minutes_" + str(x) for x in range(1, 60)) + header += ",run_minutes_60_or_greater" + print(header) sys.exit(0) # Allow "-" to be specified as file for stdin. @@ -74,6 +84,8 @@ else: jsonData = json.load(jsonFile) jsonFile.close() +timestamp = datetime.datetime.utcnow() + historyData = jsonData['dishGetHistory'] # 'current' is the count of data samples written to the ring buffer, @@ -103,6 +115,11 @@ totObstruct = 0 totObstructD = 0 totObstructOne = 0 +secondRuns = [0] * 60 +minuteRuns = [0] * 60 +runLength = 0 +initRun = None + if fParseAll or nSamples < parseSamples: parseSamples = nSamples @@ -119,6 +136,18 @@ for i in sampleRange: tot += d if d >= 1: totOne += d + runLength += 1 + elif runLength > 0: + if initRun is None: + initRun = runLength + else: + if runLength <= 60: + secondRuns[runLength-1] += runLength + else: + minuteRuns[min((runLength-1)//60-1, 59)] += runLength + runLength = 0 + elif initRun is None: + initRun = 0 if not historyData["scheduled"][i]: totUnsched += 1 totUnschedD += d @@ -130,6 +159,14 @@ for i in sampleRange: if d >= 1: totObstructOne += d +# If the entire sample set is one big drop run, it will be both initial +# fragment (continued from prior sample range) and final one (continued +# to next sample range), but to avoid double-reporting, just call it +# the initial run. +if initRun is None: + initRun = runLength + runLength = 0 + if fVerbose: print("Parsed samples: " + str(parseSamples)) print("Total ping drop: " + str(tot)) @@ -140,6 +177,14 @@ if fVerbose: print("Unscheduled: " + str(totUnsched)) print("Unscheduled ping drop: " + str(totUnschedD)) print("Unscheduled drop == 1: " + str(totUnschedOne)) + if fRunLengths: + print("Initial drop run fragment: " + str(initRun)) + print("Final drop run fragment: " + str(runLength)) + print("Per-second drop runs: " + ", ".join(str(x) for x in secondRuns)) + print("Per-minute drop runs: " + ", ".join(str(x) for x in minuteRuns)) else: # NOTE: When changing data output format, also change the -H header printing above. - print(datetime.datetime.utcnow().replace(microsecond=0).isoformat()+","+str(parseSamples)+","+str(tot)+","+str(totOne)+","+str(totObstruct)+","+str(totObstructD)+","+str(totObstructOne)+","+str(totUnsched)+","+str(totUnschedD)+","+str(totUnschedOne)) + csvData = timestamp.replace(microsecond=0).isoformat() + "," + ",".join(str(x) for x in [parseSamples, tot, totOne, totObstruct, totObstructD, totObstructOne, totUnsched, totUnschedD, totUnschedOne]) + if fRunLengths: + csvData += "," + ",".join(str(x) for x in chain([initRun, runLength], secondRuns, minuteRuns)) + print(csvData)