This section contains Probability Density Function (PDF) plots calculated from Power Spectral Densities (PSD). The first two plots display PDFs " - f'for the stations with the highest and lowest median daily-RMS variance (sample_rms, scaled by metadata sensitivity), ' - f'with an additional criteria that omits station-channels with less than 75% data availability (sample_rms, scaled by metadata sensitivity), ' + f'with an additional criteria that omits station-channels with less than 75% data availability ({availabilityType} metric, ' f"calculated between the first and last timestamp of the data for that channel), unless none of the stations have at least 75% availability. " f"These plots are intended to illustrate the range of characteristic noise levels across the experiment, avoiding stations with significant " @@ -1941,7 +2028,7 @@ def doReport( f"errors. The third PDF is a composite plot of all stations for each channel set, giving an overview of the most common noise levels for the " f"experiment as a whole.
" f"Detailed information about these PDF plots and how MUSTANG generates them " - f'can be found by visiting the ' + f'can be found by visiting the ' f"noise-pdf web service.
" f"Please note that you can also click the PDF Browser links below to access the MUSTANG PDF Browser, which allows you to view PDF plots " f"for all stations in the network on a total, annual, monthly, daily, or custom time period basis. Within the Browser, you can " @@ -1952,7 +2039,7 @@ def doReport( spectrgramPlotIntoText = ( f"The daily-PDF-mode spectrogram plots in this section illustrate the power spectra values across time for two stations with high data " f"availability and continuity. For the purpose of showing potentially different station behavior, they are secondarily selected for " - f'differing median daily-RMS variance (sample_rms ' + f'differing median daily-RMS variance (sample_rms ' f" metric, scaled by metadata sensitivity). For each channel group, the station selection steps are: " f"
Detailed information about these spectrogram plots and how MUSTANG " - f'generates them can be found by visiting the noise-spectrogram web service.
' f"Please note that you can also click the Spectrogram Browser links below to access the MUSTANG spectrogram " f"browser for each channel group below, which allows you to view spectrogram plots from all the stations in your network.
" @@ -2274,7 +2361,7 @@ def doReport( "MUSTANG is the Quality Assurance system at the IRIS DMC. It contains around 45 metrics related to the quality of data in the archives there.\n\n" ) f.write( - 'The majority of metrics are available via the measurements web service.\n\n' + 'The majority of metrics are available via the measurements web service.\n\n' ) f.write( 'To learn more about the metrics, navigate to the measurements service Service Interface page and hit the red "Current List of all metrics" for a brief description and links to more detailed documentation.\n\n' @@ -2293,7 +2380,7 @@ def doReport( if metric == "scale_corrected_sample_rms": metric = "sample_rms" metricsLinks = ( - f"http://service.iris.edu/mustang/measurements/1/query?metric={metric}" + f"http://service.earthscope.org/mustang/measurements/1/query?metric={metric}" f"&network={network}&station={stations}&location={locations}&channel={channels}" f"&start={startDate}&end={endDate}&format=text" ) @@ -2331,7 +2418,7 @@ def doReport( nStations = 1 pdfLink = ( - f"http://service.iris.edu/mustang/noise-pdf-browser/1/gallery?" + f"http://service.earthscope.org/mustang/noise-pdf-browser/1/gallery?" f"network={network}&channel={channel[0:2]}?&interval=all&" f"starttime={startDate}&endtime={endDate}" ) @@ -2418,7 +2505,7 @@ def doReport( spectPowerRange = ",".join([str(int) for int in userDefinedPowerRange]) spectLink = ( - f"http://service.iris.edu/mustang/noise-pdf-browser/1/spectrogram?" + f"http://service.earthscope.org/mustang/noise-pdf-browser/1/spectrogram?" f"network={network}&channel={channel[0:2]}?&" f"starttime={startDate}&endtime={endDate}&color.palette={spectColorPalette}&powerrange={spectPowerRange}" ) @@ -2594,7 +2681,7 @@ def main(): --metric[s]=: comma-separated list of metrics to run for the boxplots; defaults: {','.join(metricList)} --maxplot=: number of stations to include in the boxplots; defaults to {nBoxPlotSta} --colorpalette=: color palette for spectrograms; defaults to '{spectColorPalette}' - options available at http://service.iris.edu/mustang/noise-spectrogram/1/ + options available at http://service.earthscope.org/mustang/noise-spectrogram/1/ --includeoutliers=: whether to include outliers in the boxplots, True/False; defaults to {includeOutliers} --spectralrange=: power range to use in the PDFs and spectrograms, comma separated values: min,max; defaults depend on channel type --basemap=: the name of the basemap to be used for the map; defaults to '{basemap}' @@ -2753,22 +2840,32 @@ def main(): nTop = int(nBoxPlotSta / 2) nBottom = int(nBoxPlotSta - nTop) + # Check independently whether the ts_ (time-series) MUSTANG metrics exist right now - + # this can go down (or come back) on its own schedule, separate from the fdsnws-availability service + tsMetricsExist = reportUtils.checkTsMetricsExist(startDate, endDate) + # Create Availability Plots - [topStationsDict, splitPlots, avFilesDictionary, services, availabilityType] = ( - doAvailability( - splitPlots, - startDate, - endDate, - network, - stations, - locations, - channels, - nBoxPlotSta, - nTop, - nBottom, - tolerance, - imageDir, - ) + [ + topStationsDict, + splitPlots, + avFilesDictionary, + services, + availabilityType, + availabilityExistsByService, + ] = doAvailability( + splitPlots, + startDate, + endDate, + network, + stations, + locations, + channels, + nBoxPlotSta, + nTop, + nBottom, + tolerance, + imageDir, + tsMetricsExist, ) # Create BoxPlots @@ -2788,6 +2885,7 @@ def main(): nBottom, includeOutliers, imageDir, + tsMetricsExist, ) ) @@ -2806,6 +2904,8 @@ def main(): network, stations, locations, + availabilityExistsByService, + tsMetricsExist, ) # Create Spectrogram Plots @@ -2821,6 +2921,8 @@ def main(): powerRanges, spectColorPalette, imageDir, + availabilityExistsByService, + tsMetricsExist, ) # Generate Station Map diff --git a/README.md b/README.md index 9f91b28..32745c7 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ perform installs, updates, or run PIQQA. ``` cd piqqa conda update conda -conda create --name piqqa python=3.9 -c conda-forge --file piqqa-conda-install.txt +conda create --name piqqa python=3.12 -c conda-forge --file piqqa-conda-install.txt conda activate piqqa ``` @@ -122,7 +122,7 @@ Running PIQQA.py without any arguments, or with the -h/--h flags, will produce s --metrics=: comma-separated list of metrics to run for the boxplots; defaults: ts_channel_continuity,sample_rms,ts_num_gaps --maxplot=: number of stations to include in the boxplots; defaults to 30 --colorpalette=: color palette for spectrograms; defaults to 'RdYlBu' - options available at http://service.iris.edu/mustang/noise-spectrogram/1/ + options available at http://service.earthscope.org/mustang/noise-spectrogram/1/ --includeoutliers=: whether to include outliers in the boxplots, True/False; defaults to False --spectralrange=: power range to use in the PDFs and spectrograms, comma separated values: min, max; defaults depend on channel type --basemap=: the name of the basemap to be used for the map; defaults to 'stamen-terrain' @@ -149,7 +149,7 @@ In addition to those fields, there are a number of optional fields. `maxplot` is how you can limit the number of stations displayed in the boxplots. -`colorpalette` determines the color palette used in the PDFs and spectrograms, and a list of all options can be found in the service interface page for the [noise-spectrgram](http://service.iris.edu/mustang/noise-spectrogram/1/) service +`colorpalette` determines the color palette used in the PDFs and spectrograms, and a list of all options can be found in the service interface page for the [noise-spectrgram](http://service.earthscope.org/mustang/noise-spectrogram/1/) service `includeoutliers` toggles the outliers on/off in the boxplots. Generally, it is easier to view the boxplots with the outliers turned off, but there may be cases where they should be displayed. diff --git a/piqqa-conda-install.txt b/piqqa-conda-install.txt index d7ec76f..29e7fa9 100644 --- a/piqqa-conda-install.txt +++ b/piqqa-conda-install.txt @@ -1,5 +1,5 @@ -pandas=1.3.0 -numpy=1.21.0 -requests= 2.25.1 -matplotlib= 3.4.2 -plotly +pandas=2.1.4 +numpy=1.26.3 +requests=2.31.0 +matplotlib=3.8.2 +plotly=6.0.1 diff --git a/reportUtils.py b/reportUtils.py index 9406d3a..5b5ef3b 100644 --- a/reportUtils.py +++ b/reportUtils.py @@ -1,8 +1,8 @@ -''' +""" :copyright: Copyright (C) 2021 Laura Keyson, IRIS Data Management Center - -:license: + +:license: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -18,464 +18,822 @@ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations -under the License. - -''' +under the License. +""" import requests from io import StringIO import pandas as pd import numpy as np import time -import urllib +import urllib.request + +USER_AGENT = "earthscope-piqqa" + +# pandas' read_csv falls back to urllib for plain http(s) URLs (no fsspec installed), +# so installing a default opener here also sets the User-Agent for every pd.read_csv(url) +# call and any bare urllib.request.urlopen() call in this process, not just SESSION.get(). +_opener = urllib.request.build_opener() +_opener.addheaders = [("User-Agent", USER_AGENT)] +urllib.request.install_opener(_opener) + +SESSION = requests.Session() +SESSION.headers.update({"User-Agent": USER_AGENT}) + + +def get_extents_from_mustang(snclqs, startDate, endDate): + availabilityDF = pd.DataFrame() + for snclq in snclqs: + staloc = ( + snclq.split(".")[1] + + "." + + (snclq.split(".")[2] if snclq.split(".")[2] != "" else "--") + ) + URL = ( + f"https://service.earthscope.org/mustang/measurements/1/" + f"query?metric=percent_availability&target={snclq}&format=text&timewindow={startDate},{endDate}&nodata=404" + ) + + try: + tempDF = pd.read_csv(URL, header=1, parse_dates=["start", "end"]) + + # find the first and last non-0 availability values and use those as the start and end times + nonZeroDF = tempDF[tempDF["value"] > 0] + if not nonZeroDF.empty: + startTime = nonZeroDF["start"].min() + endTime = nonZeroDF["end"].max() + tempDF = pd.DataFrame( + { + "network": [snclq.split(".")[0]], + "station": [snclq.split(".")[1]], + "location": [snclq.split(".")[2]], + "channel": [snclq.split(".")[3]], + "quality": [snclq.split(".")[4]], + "staloc": [staloc], + "earliest": [startTime], + "latest": [endTime], + } + ) + else: + tempDF = pd.DataFrame() + + availabilityDF = pd.concat([availabilityDF, tempDF], ignore_index=True) + except Exception as e: + print( + f" INFO: Unable to retrieve availability for {snclq} from Mustang service" + ) + print(f" {URL}") + pass + + return availabilityDF + + +def mergeGapsWithinTolerance(availabilityDF, tolerance): + """Merge adjacent time spans (per network/station/location/channel/quality) that are + separated by a gap no larger than `tolerance` seconds. The availability service used + to do this server-side via the mergegaps parameter; the migrated service no longer + honors it, so this reproduces the same gap-tolerance behavior client-side instead.""" + if availabilityDF.empty: + return availabilityDF + + tolerance = float(tolerance) + groupCols = [ + c + for c in ["network", "station", "location", "channel", "quality"] + if c in availabilityDF.columns + ] + if not groupCols: + return availabilityDF + + mergedRows = [] + for _, group in availabilityDF.groupby(groupCols, dropna=False): + group = group.sort_values("earliest") + current = None + for _, row in group.iterrows(): + if current is None: + current = row.copy() + continue + gapSeconds = (row["earliest"] - current["latest"]).total_seconds() + if gapSeconds <= tolerance: + if row["latest"] > current["latest"]: + current["latest"] = row["latest"] + else: + mergedRows.append(current) + current = row.copy() + if current is not None: + mergedRows.append(current) + + return pd.DataFrame(mergedRows).reset_index(drop=True) def getAvailability(snclqs, startDate, endDate, tolerance, avtype): availabilityDF = pd.DataFrame() services = [] - - - if avtype == '': + + if avtype == "": for snclq in snclqs: - snclqList = snclq.split('.') - n =snclqList[0] + snclqList = snclq.split(".") + n = snclqList[0] s = snclqList[1] l = snclqList[2] - if l == '': - luse = '--' + if l == "": + luse = "--" else: luse = l c = snclqList[3] q = snclqList[4] - + if q == "M": service = "fdsnws" elif q == "D": service = "ph5ws" - + if service not in services: services.append(service) - URL = f'http://service.iris.edu/{service}/availability/1/query?format=text&' \ - f'net={n}&sta={s}&loc={luse}&cha={c}&quality={q}&' \ - f'starttime={startDate}&endtime={endDate}&orderby=nslc_time_quality_samplerate&' \ - f'mergegaps={tolerance}&includerestricted=true&nodata=404' - try: - tmpDF = pd.read_csv(URL, sep=' ', dtype={'Location': str, 'Station': str}, parse_dates=['Earliest','Latest']) - tmpDF['staloc'] = f'{s}.{luse}' - availabilityDF = availabilityDF.append(tmpDF, ignore_index=True) + URL = ( + f"http://service.earthscope.org/{service}/availability/1/query?format=text&" + f"net={n}&sta={s}&loc={luse}&cha={c}&quality={q}&" + f"starttime={startDate}&endtime={endDate}&includerestricted=true&nodata=404" + ) + try: + tmpDF = pd.read_csv( + URL, + sep=" ", + dtype={"Location": str, "Station": str}, + parse_dates=["Earliest", "Latest"], + ) + tmpDF["staloc"] = f"{s}.{luse}" + availabilityDF = pd.concat([availabilityDF, tmpDF], ignore_index=True) except Exception as e: pass - - - elif avtype == 'extents': + + elif avtype == "extents": # Don't loop over the stations unless necessary - nets = ','.join(list(set([n.split('.')[0] for n in snclqs]))) - stas = ','.join(list(set([n.split('.')[1] for n in snclqs]))) + nets = ",".join(list(set([n.split(".")[0] for n in snclqs]))) + stas = ",".join(list(set([n.split(".")[1] for n in snclqs]))) stas = "*" - locs = ','.join(list(set([n.split('.')[2] for n in snclqs]))) + locs = ",".join(list(set([n.split(".")[2] for n in snclqs]))) if locs == "": - locs = '*' - chans = ','.join(list(set([n.split('.')[3] for n in snclqs]))) - qs = ','.join(list(set([n.split('.')[4] for n in snclqs]))) - - if qs == 'D': - service = 'ph5ws' - elif qs == 'M': - service = 'fdsnws' - + locs = "*" + chans = ",".join(list(set([n.split(".")[3] for n in snclqs]))) + qs = ",".join(list(set([n.split(".")[4] for n in snclqs]))) + + if qs == "D": + service = "ph5ws" + elif qs == "M": + service = "fdsnws" + if service not in services: services.append(service) - - URL = f'http://service.iris.edu/{service}/availability/1/extent?format=text&' \ - f'net={nets}&sta={stas}&loc={locs}&cha={chans}&quality={qs}&' \ - f'starttime={startDate}&endtime={endDate}&orderby=nslc_time_quality_samplerate&' \ - f'includerestricted=true&nodata=404' + + URL = ( + f"http://service.earthscope.org/{service}/availability/1/extent?format=text&" + f"net={nets}&sta={stas}&loc={locs}&cha={chans}&quality={qs}&" + f"starttime={startDate}&endtime={endDate}&includerestricted=true&nodata=404" + ) try: - availabilityDF = pd.read_csv(URL, sep=' ', dtype={'Location': str, 'Station': str}, parse_dates=['Earliest','Latest']) + availabilityDF = pd.read_csv( + URL, + sep=" ", + dtype={"Location": str, "Station": str}, + parse_dates=["Earliest", "Latest"], + ) except: - print(" INFO: Unable to retrieve availability in one go, trying to loop over stations instead") - + print( + " INFO: Unable to retrieve availability in one go, trying to loop over stations instead" + ) + for snclq in snclqs: - snclqList = snclq.split('.') - n =snclqList[0] + snclqList = snclq.split(".") + n = snclqList[0] s = snclqList[1] l = snclqList[2] - if l == '': - luse = '--' + if l == "": + luse = "--" else: luse = l c = snclqList[3] q = snclqList[4] - + if q == "M": service = "fdsnws" elif q == "D": service = "ph5ws" - + if service not in services: services.append(service) - - URL = f'http://service.iris.edu/{service}/availability/1/extent?format=text&' \ - f'net={n}&sta={s}&loc={luse}&cha={c}&quality={q}&' \ - f'starttime={startDate}&endtime={endDate}&orderby=nslc_time_quality_samplerate&' \ - f'&includerestricted=true&nodata=404' - - try: - tmpDF = pd.read_csv(URL, sep=' ', dtype={'Location': str, 'Station': str}, parse_dates=['Earliest','Latest']) - tmpDF['staloc'] = f'{s}.{luse}' - availabilityDF = availabilityDF.append(tmpDF, ignore_index=True) + + URL = ( + f"http://service.earthscope.org/{service}/availability/1/extent?format=text&" + f"net={n}&sta={s}&loc={luse}&cha={c}&quality={q}&" + f"starttime={startDate}&endtime={endDate}&includerestricted=true&nodata=404" + ) + + try: + tmpDF = pd.read_csv( + URL, + sep=" ", + dtype={"Location": str, "Station": str}, + parse_dates=["Earliest", "Latest"], + ) + tmpDF["staloc"] = f"{s}.{luse}" + availabilityDF = pd.concat( + [availabilityDF, tmpDF], ignore_index=True + ) except: pass - - - - - - -# availabilityDF = availabilityDF.apply(lambda x: x.str.strip() if x.dtype == "object" else x) + + # availabilityDF = availabilityDF.apply(lambda x: x.str.strip() if x.dtype == "object" else x) availabilityDF.rename(columns=lambda x: x.strip().lower(), inplace=True) - availabilityDF.rename(columns = {'#network': 'network'}, inplace=True) - - return availabilityDF, services - + availabilityDF.rename(columns={"#network": "network"}, inplace=True) + + if avtype == "": + # The availability service no longer merges gaps server-side (the mergegaps + # parameter is a no-op on the migrated service), so do it here instead. + availabilityDF = mergeGapsWithinTolerance(availabilityDF, tolerance) + + return availabilityDF, services + + +def getDataStartEnd( + target, + thisNetwork, + thisStation, + thisLocation, + thisChannel, + startDate, + endDate, + tolerance, + availabilityExistsByService, +): + """Get the start/end times of the data for a target, via the availability + service if it's up, otherwise falling back to MUSTANG extents. Falls back + to the report's own startDate/endDate if no start/end can be determined + either way (e.g. percent_availability is also down or has no data). + availabilityExistsByService is a dict like {"fdsnws": True, "ph5ws": False} - + the target's own quality code (its last dot-component) picks which entry applies, + since fdsnws-availability and ph5ws-availability can go down independently.""" + reportStart = f"{startDate}T00:00:00" + reportEnd = f"{endDate}T00:00:00" + + serviceKey = serviceForQuality(target.split(".")[-1]) + availabilityExists = availabilityExistsByService.get(serviceKey, False) + + if availabilityExists: + thisStationAvDF, _ = getAvailability( + [target], startDate, endDate, tolerance, "" + ) + if thisStationAvDF.empty: + return reportStart, reportEnd + + thisAvail = thisStationAvDF[ + (thisStationAvDF["station"] == thisStation) + & (thisStationAvDF["network"] == thisNetwork) + & (thisStationAvDF["location"] == thisLocation) + & (thisStationAvDF["channel"] == thisChannel) + ] + if thisAvail.empty: + return reportStart, reportEnd + + thisStart = thisAvail["earliest"].dt.strftime("%Y-%m-%dT%H:%M:%S").min() + thisEnd = thisAvail["latest"].dt.strftime("%Y-%m-%dT%H:%M:%S").max() + else: + thisStationAvDF = get_extents_from_mustang([target], startDate, endDate) + if thisStationAvDF.empty: + print( + f" WARNING: Unable to determine data start/end for {target} " + f"(percent_availability returned no data from Mustang), using report timespan instead" + ) + return reportStart, reportEnd + + thisStart = thisStationAvDF["earliest"].dt.strftime("%Y-%m-%dT%H:%M:%S").min() + thisEnd = thisStationAvDF["latest"].dt.strftime("%Y-%m-%dT%H:%M:%S").max() + + return thisStart, thisEnd + def retrieveMetrics(URL, metric): - response = requests.get(URL) + response = SESSION.get(URL) tempDF = pd.read_csv(StringIO(response.text), header=1) - tempDF.rename(columns={'target':'snclq'}, inplace=True) - tempDF['target'] = tempDF['snclq'].apply(lambda x: '.'.join(x.split('.')[0:4])) - tempDF['station'] = tempDF['snclq'].apply(lambda x: '.'.join(x.split('.')[1:3])) ## Because "station" is really "station.location" - tempDF['station'] = [x + '--' if x.endswith('.') else x for x in tempDF['station'] ] - - if (not metric == 'transfer_function') and (not metric == 'orientation_check'): - tempDF.rename(columns = {'value': metric}, inplace=True) - tempDF[metric] = tempDF[metric].map(float) - tempDF.drop('lddate', axis=1, inplace=True) - - tempDF['start'] = pd.to_datetime(tempDF['start']) - tempDF['end'] = pd.to_datetime(tempDF['end']) - + tempDF.rename(columns={"target": "snclq"}, inplace=True) + tempDF["target"] = tempDF["snclq"].apply(lambda x: ".".join(x.split(".")[0:4])) + tempDF["station"] = tempDF["snclq"].apply( + lambda x: ".".join(x.split(".")[1:3]) + ) ## Because "station" is really "station.location" + tempDF["station"] = [x + "--" if x.endswith(".") else x for x in tempDF["station"]] + + if (not metric == "transfer_function") and (not metric == "orientation_check"): + tempDF.rename(columns={"value": metric}, inplace=True) + tempDF[metric] = tempDF[metric].map(float) + tempDF.drop("lddate", axis=1, inplace=True) + + tempDF["start"] = pd.to_datetime(tempDF["start"]) + tempDF["end"] = pd.to_datetime(tempDF["end"]) + if metric == "sample_rms" and tempDF.empty: print(f" --> Unable to get measurements for {metric}") print(f" {URL}") - + return tempDF -def addMetricToDF(metric, DF, network, stations, locations, channels, startDate, endDate): - if not (metric== 'ts_percent_availability_total' or metric == 'percent_availability'): + +def checkTsMetricsExist(startDate, endDate, timeout=15): + # Probe against a long-running reference station (IU.ANMO) rather than the + # network being reported on, so the result reflects whether ts_ metrics exist + # at all rather than whether this particular network has data for them. + testURL = ( + f"https://service.earthscope.org/mustang/measurements/1/query?" + f"metric=ts_percent_availability_total&net=IU&sta=ANMO&loc=00&chan=BHZ&" + f"format=text&timewindow={startDate},{endDate}&nodata=404" + ) + try: + response = SESSION.get(testURL, timeout=timeout) + if "Could not find metric" in response.text: + return False + return True + except Exception: + return False + + +def addMetricToDF( + metric, DF, network, stations, locations, channels, startDate, endDate +): + if not ( + metric == "ts_percent_availability_total" or metric == "percent_availability" + ): print(f" Retrieving {metric}") chanList = list() - for chan in channels.split(','): - if (len(chan) == 3) and (chan[2] in ['*', '?', '.', '_']): - chan = f'{chan[0:2]}Z' + for chan in channels.split(","): + if (len(chan) == 3) and (chan[2] in ["*", "?", ".", "_"]): + chan = f"{chan[0:2]}Z" elif len(chan) == 2: chan = f"{chan}Z" elif chan == "*": chan = "??Z" chanList.append(chan) - URL = f"http://service.iris.edu/mustang/measurements/1/query?metric={metric}&net={network}&" \ - f"sta={stations}&loc={locations}&chan={','.join(chanList)}" \ - f'&format=text&timewindow={startDate},{endDate}&nodata=404' - - + URL = ( + f"http://service.earthscope.org/mustang/measurements/1/query?metric={metric}&net={network}&" + f"sta={stations}&loc={locations}&chan={','.join(chanList)}" + f"&format=text&timewindow={startDate},{endDate}&nodata=404" + ) + try: tempDF = retrieveMetrics(URL, metric) except Exception as e: - if not metric== 'ts_percent_availability_total': - print(f" --> Unable to get measurements for {metric}, waiting 5 seconds and trying again") + if not metric == "ts_percent_availability_total": + print( + f" --> Unable to get measurements for {metric}, waiting 5 seconds and trying again" + ) print(f" {URL}") -# print(f" {e}") + # print(f" {e}") time.sleep(5) try: tempDF = retrieveMetrics(URL, metric) except: - if not metric== 'ts_percent_availability_total': - print(f" --> Still unable to get measurements for {metric}, bypassing" ) + if not metric == "ts_percent_availability_total": + print( + f" --> Still unable to get measurements for {metric}, bypassing" + ) tempDF = pd.DataFrame() - - + if DF.empty: DF = tempDF.copy() else: try: - DF = pd.merge(DF, tempDF, how='outer', left_on=['target','snclq', 'station', 'start', 'end'], right_on=['target','snclq','station', 'start', 'end']) + DF = pd.merge( + DF, + tempDF, + how="outer", + left_on=["target", "snclq", "station", "start", "end"], + right_on=["target", "snclq", "station", "start", "end"], + ) except: print(f" ERROR: Something went wrong with the {metric}") - + return DF - + + +def serviceForQuality(quality): + # PH5/PASSCAL data (quality "D") is served by ph5ws (service.iris.edu), while + # regular FDSN data (quality "M") is served by fdsnws (service.earthscope.org) - + # these can go down independently of each other. + return "ph5ws" if quality == "D" else "fdsnws" + + +def getAvailabilityServiceURLs( + network, stations, locations, channels, startDate, endDate +): + """Determine which availability service(s) - fdsnws and/or ph5ws - actually apply + to this network via FedCatalog, so callers can check/use the correct one(s) instead + of assuming fdsnws. Returns a dict like {"fdsnws": "https://.../availability/1/"}. + """ + chanList = list() + for chan in channels.split(","): + if (len(chan) == 3) and (chan[2] in ["*", "?", ".", "_"]): + chan = f"{chan[0:2]}Z" + elif len(chan) == 2: + chan = f"{chan}Z" + elif chan == "*": + chan = "??Z" + chanList.append(chan) + + fedURL = ( + f"http://service.earthscope.org/irisws/fedcatalog/1/query?" + f"net={network}&sta={stations}&loc={locations}&cha={','.join(chanList)}&" + f"starttime={startDate}&endtime={endDate}" + f"&format=request&includeoverlaps=false" + ) + + serviceURLs = {} + try: + with urllib.request.urlopen(fedURL) as response: + html_content = response.read().decode("utf-8") + for ln in html_content.split("\n"): + if ln.startswith("AVAILABILITYSERVICE="): + url = ln.split("=")[1] + if "ph5ws" in url: + serviceURLs["ph5ws"] = url + elif "fdsnws" in url: + serviceURLs["fdsnws"] = url + except Exception as e: + print( + f" ERROR: unable to retrieve fed catalog information about where the data lives - {fedURL}\n{e}" + ) + # Fall back to checking both - each is probed independently, so this is conservative, not incorrect. + serviceURLs = { + "fdsnws": "https://service.earthscope.org/fdsnws/availability/1/", + "ph5ws": "http://service.iris.edu/ph5ws/availability/1/", + } + + return serviceURLs + + +def checkAvailabilityServicesExist( + network, stations, locations, channels, startDate, endDate +): + """Probe each availability service relevant to this network independently, since + fdsnws-availability and ph5ws-availability can go down independently of each other. + Returns a dict like {"fdsnws": True, "ph5ws": False} covering only the service(s) + actually relevant to this network.""" + serviceURLs = getAvailabilityServiceURLs( + network, stations, locations, channels, startDate, endDate + ) + availabilityExistsByService = {} + for serviceKey, url in serviceURLs.items(): + try: + urllib.request.urlopen(url) + availabilityExistsByService[serviceKey] = True + except urllib.error.HTTPError as e: + if e.code == 410: + print( + f"WARNING: {serviceKey}-availability service is down, falling back to MUSTANG percent_availability for data extents" + ) + else: + print( + f"WARNING: {serviceKey}-availability service returned HTTP {e.code}, falling back to MUSTANG percent_availability for data extents" + ) + availabilityExistsByService[serviceKey] = False + except Exception as e: + print( + f"WARNING: {serviceKey}-availability service is down ({e}), falling back to MUSTANG percent_availability for data extents" + ) + availabilityExistsByService[serviceKey] = False + + return availabilityExistsByService + + def getMetadata(network, stations, locations, channels, startDate, endDate, level): # This one and getStations are almost redundant, except that they return at different # levels. Merge into one function that also takes level as an input? - - if level == 'channel': + + if level == "channel": stationDF = pd.DataFrame() - if level == 'station': - stationDF = pd.DataFrame(columns=['#Network', 'Station', 'Latitude' , 'Longitude' , 'Elevation' , 'SiteName' , 'StartTime' , 'EndTime']) - if level == 'network': - stationDF = pd.DataFrame(columns=['#Network',]) - + if level == "station": + stationDF = pd.DataFrame( + columns=[ + "#Network", + "Station", + "Latitude", + "Longitude", + "Elevation", + "SiteName", + "StartTime", + "EndTime", + ] + ) + if level == "network": + stationDF = pd.DataFrame( + columns=[ + "#Network", + ] + ) + chanList = list() - for chan in channels.split(','): - if (len(chan) == 3) and (chan[2] in ['*', '?', '.', '_']): - chan = f'{chan[0:2]}Z' + for chan in channels.split(","): + if (len(chan) == 3) and (chan[2] in ["*", "?", ".", "_"]): + chan = f"{chan[0:2]}Z" elif len(chan) == 2: -# chan = f"{chan}Z,{chan}1" + # chan = f"{chan}Z,{chan}1" chan = f"{chan}Z" elif chan == "*": -# chan = "??Z,??1" + # chan = "??Z,??1" chan = "??Z" chanList.append(chan) - try: - # Call Fed Catalog to know what service the network can be retrieved using. + # Call Fed Catalog to know what service the network can be retrieved using. print(" Calling on Fed Catalog") - fedURL = f"http://service.iris.edu/irisws/fedcatalog/1/query?" \ - f"net={network}&sta={stations}&loc={locations}&cha={','.join(chanList)}&" \ - f"starttime={startDate}&endtime={endDate}" \ - f"&format=request&includeoverlaps=false" - + fedURL = ( + f"http://service.earthscope.org/irisws/fedcatalog/1/query?" + f"net={network}&sta={stations}&loc={locations}&cha={','.join(chanList)}&" + f"starttime={startDate}&endtime={endDate}" + f"&format=request&includeoverlaps=false" + ) try: with urllib.request.urlopen(fedURL) as response: - html_content = response.read().decode('utf-8') - + html_content = response.read().decode("utf-8") + services = [] - for ln in html_content.split('\n'): + for ln in html_content.split("\n"): if ln.startswith("STATIONSERVICE="): - serviceURL = ln.split('=')[1] - if 'iris' in serviceURL: + serviceURL = ln.split("=")[1] + if "earthscope" in serviceURL or "iris" in serviceURL: services.append(serviceURL) - + except Exception as e: - print(" ERROR: unable to retrieve fed catalog information about where the data lives - %s\n%s " % (fedURL, e)) - services = ['http://service.iris.edu/fdsnws/station/1/', 'http://service.iris.edu/ph5ws/station/1/'] - + print( + " ERROR: unable to retrieve fed catalog information about where the data lives - %s\n%s " + % (fedURL, e) + ) + services = [ + "http://service.earthscope.org/fdsnws/station/1/", + "http://service.earthscope.org/ph5ws/station/1/", + ] + for service in services: # To prevent needing to know a priori where it's from, try both and only add if attempt is successful # Most experiments are one-archive only, but some have been split in the past + tmpDF = pd.DataFrame() try: print(" Calling on Station Service") - stationURL = f"{service}query?" \ - f"net={network}&sta={stations}&loc={locations}&cha={','.join(chanList)}&" \ - f"starttime={startDate}&endtime={endDate}&level={level}" \ - f"&format=text&includecomments=true&nodata=404" + stationURL = ( + f"{service}query?" + f"net={network}&sta={stations}&loc={locations}&cha={','.join(chanList)}&" + f"starttime={startDate}&endtime={endDate}&level={level}" + f"&format=text&includecomments=true&nodata=404" + ) - if level == 'channel': + if level == "channel": try: - tmpDF = pd.read_csv(stationURL, sep='|', dtype={' Location ': str, ' Station ': str}) + tmpDF = pd.read_csv( + stationURL, + sep="|", + dtype={" Location ": str, " Station ": str}, + ) tmpDF.rename(columns=lambda x: x.strip(), inplace=True) - tmpDF.rename(columns = {'#Network': 'Network'}, inplace=True) - tmpDF['Location'] = tmpDF.Location.replace(np.nan, '', regex=True) - tmpDF['Target'] = tmpDF[['Network', 'Station', 'Location','Channel']].apply(lambda x: '.'.join(x.map(str)), axis=1) + tmpDF.rename(columns={"#Network": "Network"}, inplace=True) + tmpDF["Location"] = tmpDF.Location.replace( + np.nan, "", regex=True + ) + tmpDF["Target"] = tmpDF[ + ["Network", "Station", "Location", "Channel"] + ].apply(lambda x: ".".join(x.map(str)), axis=1) tmpDF.columns = tmpDF.columns.str.lower() - - tmpDF['starttime'] = pd.to_datetime(tmpDF['starttime']) - tmpDF['endtime'] = pd.to_datetime(tmpDF['endtime']) - + + tmpDF["starttime"] = pd.to_datetime(tmpDF["starttime"]) + tmpDF["endtime"] = pd.to_datetime(tmpDF["endtime"]) + except Exception as e: - print(f" ERROR: Unable to retrieve channel information from {stationURL}") - - elif level == 'station' or level == 'network': + print( + f" ERROR: Unable to retrieve channel information from {stationURL}" + ) + tmpDF = pd.DataFrame() + + elif level == "station" or level == "network": try: - tmpDF = pd.read_csv(stationURL, sep='|') + tmpDF = pd.read_csv(stationURL, sep="|") tmpDF.rename(columns=lambda x: x.strip(), inplace=True) except: - print(f" ERROR: Unable to retrieve metadata information from {stationURL}") + print( + f" ERROR: Unable to retrieve metadata information from {stationURL}" + ) + tmpDF = pd.DataFrame() except: tmpDF = pd.DataFrame() - - stationDF = pd.concat([stationDF, tmpDF], ignore_index=True) + + if tmpDF.empty: + continue + if stationDF.empty: + stationDF = tmpDF.copy() + else: + stationDF = pd.concat([stationDF, tmpDF], ignore_index=True) except: - print(" ERROR: Unable to retrieve metadata") + print(" ERROR: Unable to retrieve metadata") return stationDF - + return stationDF + def retrieveExpectedPDFs(NSLC, startDate, endDate): - net = NSLC.split('.')[0] - sta = NSLC.split('.')[1] - loc = NSLC.split('.')[2] + net = NSLC.split(".")[0] + sta = NSLC.split(".")[1] + loc = NSLC.split(".")[2] cha = f"{NSLC.split('.')[3]}?" - URL = f'http://service.iris.edu/mustang/noise-pdf-browser/1/availability?network={net}&station={sta}&location={loc}&channel={cha}&starttime={startDate}&endtime={endDate}&interval=all' + URL = f"http://service.earthscope.org/mustang/noise-pdf-browser/1/availability?network={net}&station={sta}&location={loc}&channel={cha}&starttime={startDate}&endtime={endDate}&interval=all" - response = requests.get(URL) + response = SESSION.get(URL) if response.text.startswith("Error"): # Wait 5 seconds and try again - print(f" --> Error retrieving list of expected PDFs for {NSLC}, waiting 5 seconds and trying again") + print( + f" --> Error retrieving list of expected PDFs for {NSLC}, waiting 5 seconds and trying again" + ) time.sleep(5) - response = requests.get(URL) + response = SESSION.get(URL) if response.text.startswith("Error"): print(f" --> Unable to retrieve PDF list for {NSLC}") -# print(response.text) + # print(response.text) expectedTargets = list() - - # doing it this way so that this section will run if either the first or second attempt was successful - if not response.text.startswith("Error"): - expectedTargets = [x.split(',')[0] for x in response.text.split('\n') if not x == ''] - + + # doing it this way so that this section will run if either the first or second attempt was successful + if not response.text.startswith("Error"): + expectedTargets = [ + x.split(",")[0] for x in response.text.split("\n") if not x == "" + ] + return expectedTargets + def getPDF(target, startDate, endDate, spectPowerRange, imageDir): - net = target.split('.')[0] - sta = target.split('.')[1] - loc = target.split('.')[2] - if loc == '': - loc = '--' - cha = target.split('.')[3] - - - plot_titlefont=20 - plot_subtitlefont=18 - plot_axisfont=16 - plot_labelfont=18 - plotArguments = f"plot.titlefont.size={plot_titlefont}&plot.subtitlefont.size={plot_subtitlefont}" \ - f"&plot.axisfont.size={plot_axisfont}&plot.labelfont.size={plot_labelfont}" - - URL = f"http://service.iris.edu/mustang/noise-pdf/1/query?&" \ - f"network={net}&station={sta}&location={loc}&channel={cha}&quality=?&" \ - f"starttime={startDate}&endtime={endDate}&format=plot&plot.interpolation=bicubic&nodata=404&" \ - f"plot.power.min={spectPowerRange[0]}&plot.power.max={spectPowerRange[1]}&{plotArguments}" - - response = requests.get(URL) - filename = (f"{imageDir}/{target}_PDF.png").replace('*','').replace('?','') - + net = target.split(".")[0] + sta = target.split(".")[1] + loc = target.split(".")[2] + if loc == "": + loc = "--" + cha = target.split(".")[3] + + plot_titlefont = 20 + plot_subtitlefont = 18 + plot_axisfont = 16 + plot_labelfont = 18 + plotArguments = ( + f"plot.titlefont.size={plot_titlefont}&plot.subtitlefont.size={plot_subtitlefont}" + f"&plot.axisfont.size={plot_axisfont}&plot.labelfont.size={plot_labelfont}" + ) + + URL = ( + f"http://service.earthscope.org/mustang/noise-pdf/1/query?&" + f"network={net}&station={sta}&location={loc}&channel={cha}&quality=?&" + f"starttime={startDate}&endtime={endDate}&format=plot&plot.interpolation=bicubic&nodata=404&" + f"plot.power.min={spectPowerRange[0]}&plot.power.max={spectPowerRange[1]}&{plotArguments}" + ) + + response = SESSION.get(URL) + filename = (f"{imageDir}/{target}_PDF.png").replace("*", "").replace("?", "") + file = open(filename, "wb") file.write(response.content) file.close() - + return filename - -def getSpectrogram(target, startDate, endDate, spectPowerRange, spectColorPalette, imageDir): - powerRange = ','.join([str(x) for x in spectPowerRange]) - plot_titlefont=20 - plot_subtitlefont=18 - plot_axisfont=16 - plot_labelfont=18 - plotArguments = f"plot.titlefont.size={plot_titlefont}&plot.subtitlefont.size={plot_subtitlefont}" \ - f"&plot.axisfont.size={plot_axisfont}&plot.labelfont.size={plot_labelfont}" - - URL = f"http://service.iris.edu/mustang/noise-spectrogram/1/query?target={target}&" \ - f"starttime={startDate}&endtime={endDate}&output=power&format=plot&plot.color.palette={spectColorPalette}&" \ - f"plot.powerscale.range={powerRange}&plot.horzaxis=time&plot.time.matchrequest=true&{plotArguments}&" \ - f"plot.time.tickunit=auto&plot.time.invert=false&plot.powerscale.show=true&plot.powerscale.orientation=horz&nodata=404" - - - response = requests.get(URL) + + +def getSpectrogram( + target, startDate, endDate, spectPowerRange, spectColorPalette, imageDir +): + powerRange = ",".join([str(x) for x in spectPowerRange]) + plot_titlefont = 20 + plot_subtitlefont = 18 + plot_axisfont = 16 + plot_labelfont = 18 + plotArguments = ( + f"plot.titlefont.size={plot_titlefont}&plot.subtitlefont.size={plot_subtitlefont}" + f"&plot.axisfont.size={plot_axisfont}&plot.labelfont.size={plot_labelfont}" + ) + + URL = ( + f"http://service.earthscope.org/mustang/noise-spectrogram/1/query?target={target}&" + f"starttime={startDate}&endtime={endDate}&output=power&format=plot&plot.color.palette={spectColorPalette}&" + f"plot.powerscale.range={powerRange}&plot.horzaxis=time&plot.time.matchrequest=true&{plotArguments}&" + f"plot.time.tickunit=auto&plot.time.invert=false&plot.powerscale.show=true&plot.powerscale.orientation=horz&nodata=404" + ) + + response = SESSION.get(URL) filename = f"{imageDir}/{target}_spectrogram.png" file = open(filename, "wb") file.write(response.content) file.close() - + return filename def getBoundsZoomLevel(bounds, mapDim): + """ + source: https://stackoverflow.com/questions/6048975/google-maps-v3-how-to-calculate-the-zoom-level-for-a-given-bounds + :param bounds: list of ne and sw lat/lon + :param mapDim: dictionary with image size in pixels + :return: zoom level to fit bounds in the visible area + """ + n_lat = bounds[0] + w_long = bounds[1] + s_lat = bounds[2] + e_long = bounds[3] + + scale = ( + 2 # adjustment to reflect MapBox base tiles are 512x512 vs. Google's 256x256 + ) + WORLD_DIM = {"height": 256 * scale, "width": 256 * scale} + ZOOM_MAX = 16 + ZOOM_MIN = 0.5 + + def latRad(lat): + sin = np.sin(lat * np.pi / 180) + radX2 = np.log((1 + sin) / (1 - sin)) / 2 + return max(min(radX2, np.pi), -np.pi) / 2 + + def zoom(mapPx, worldPx, fraction): + return np.floor(np.log(mapPx / worldPx / fraction) / np.log(2)) + + latFraction = (latRad(n_lat) - latRad(s_lat)) / np.pi + + lngDiff = e_long - w_long + lngFraction = ((lngDiff + 360) if lngDiff < 0 else lngDiff) / 360 + lat_lng_fraction = max(latFraction, lngFraction) / min(latFraction, lngFraction) + + latZoom = zoom(mapDim["height"], WORLD_DIM["height"], latFraction) + lngZoom = zoom(mapDim["width"], WORLD_DIM["width"], lngFraction) + zoomDiff = max(latZoom, lngZoom) + + if lat_lng_fraction > 10: + zoomDiff = max(latZoom, lngZoom) - min(latZoom, lngZoom) + + return min(max(zoomDiff, ZOOM_MIN), ZOOM_MAX) + - """ - source: https://stackoverflow.com/questions/6048975/google-maps-v3-how-to-calculate-the-zoom-level-for-a-given-bounds - :param bounds: list of ne and sw lat/lon - :param mapDim: dictionary with image size in pixels - :return: zoom level to fit bounds in the visible area - """ - n_lat = bounds[0] - w_long = bounds[1] - s_lat = bounds[2] - e_long = bounds[3] - - scale = 2 # adjustment to reflect MapBox base tiles are 512x512 vs. Google's 256x256 - WORLD_DIM = {'height': 256 * scale, 'width': 256 * scale} - ZOOM_MAX = 16 - ZOOM_MIN = 0.5 - - def latRad(lat): - sin = np.sin(lat * np.pi / 180) - radX2 = np.log((1 + sin) / (1 - sin)) / 2 - return max(min(radX2, np.pi), -np.pi) / 2 - - def zoom(mapPx, worldPx, fraction): - return np.floor(np.log(mapPx / worldPx / fraction) / np.log(2)) - - latFraction = (latRad(n_lat) - latRad(s_lat)) / np.pi - - lngDiff = e_long - w_long - lngFraction = ((lngDiff + 360) if lngDiff < 0 else lngDiff) / 360 - lat_lng_fraction = max(latFraction, lngFraction) / min(latFraction, lngFraction) - - latZoom = zoom(mapDim['height'], WORLD_DIM['height'], latFraction) - lngZoom = zoom(mapDim['width'], WORLD_DIM['width'], lngFraction) - zoomDiff = max(latZoom, lngZoom) - - if lat_lng_fraction > 10: - zoomDiff = max(latZoom, lngZoom) - min(latZoom, lngZoom) - - return min(max(zoomDiff, ZOOM_MIN), ZOOM_MAX) - - def getMetricLabel(metric): - metricLabels = {'amplifier_saturation':'daily flag count \n(number of occurrences)', - 'calibration_signal':'daily flag count \n(number of occurrences)', - 'clock_locked':'daily flag count \n(number of occurrences)', - 'cross_talk':'correlation coefficient \n', # no units - 'data_latency':'latency (seconds) \n', - 'dc_offset':'daily indicator of likelihood of \nDC offset shift', # no units - 'dead_channel_gsn':'indicator \n', - 'dead_channel_lin':'standard deviation of residuals (dB) \n', - 'digital_filter_charging':'daily flag count \n(number of occurrences)', - 'digitizer_clipping':'daily flag count \n(number of occurrences)', - 'event_begin':'daily flag count \n(number of occurrences)', - 'event_end':'daily flag count \n(number of occurrences)', - 'event_in_progress':'daily flag count \n(number of occurrences)', - 'feed_latency':'latency (seconds) \n', - 'gap_list':'daily gap length \n(seconds)', - 'glitches':'daily flag count \n(number of occurrences)', - 'max_gap':'daily maximum gap length \n(seconds)', - 'max_overlap':'daily overlap length \n(seconds)', - 'max_range':'daily maximum amplitude range, \nwindowed (counts)', - 'max_stalta':'daily \nshort-term average / long-term \naverage', # no units - 'missing_padded_data':'daily flag count \n(number of occurrences)', - 'num_gaps':'daily gap count \n(number of occurrences)', - 'num_overlaps':'daily overlap count \n(number of occurrences)', - 'num_spikes':'daily outlier count \n(number of occurrences)', - 'pct_above_nhnm':'daily PDF matrix above \nNew High Noise Model (%)', - 'pct_below_nlnm':'daily PDF matrix below \nNew Low Noise Model (%)', - 'percent_availability':'daily availability (%) \n', - 'polarity_check':'maximum cross-correlation \nfunction', # no units - 'pressure_effects':'daily zero-lag \ncross-correlation function', # no units - 'sample_max':'daily maximum amplitude \n(counts)', - 'sample_mean':'daily mean amplitude \n(counts)', - 'sample_median':'daily median amplitude \n(counts)', - 'sample_min':'daily minimum amplitude \n(counts)', - 'sample_rate_channel':'daily indicator \n', - 'sample_rate_resp':'daily indicator \n', - 'sample_rms':'daily root-mean-square variance (counts) \n', - 'scale_corrected_sample_rms':'daily root-mean-squared variance,\nscaled by sensitivity', - 'sample_snr':'signal-to-noise ratio \n', # no units - 'sample_unique':'daily unique sample values \n(number of occurrences)', - 'spikes':'daily flag count \n(number of occurrences)', - 'suspect_time_tag':'daily flag count \n(number of occurrences)', - 'telemetry_sync_error':'daily flag count \n(number of occurrences)', - 'timing_correction':'daily flag count \n(number of occurrences)', - 'timing_quality':'daily average timing quality (%) \n', - 'total_latency':'latency (seconds) \n', - 'ts_num_gaps':'daily gap count \n(number of occurrences)', - 'ts_num_gaps_total':'gap count \n(number of occurrences)', - 'ts_max_gap':'daily maximum gap length \n(seconds)', - 'ts_max_gap_total':'maximum gap length \n(seconds)', - 'ts_gap_length':'daily total gap length \n(seconds)', - 'ts_gap_length_total':'total gap length (seconds) \n', - 'ts_percent_availability':'daily availability (%) \n', - 'ts_percent_availability_total':'availability (%) \n', - 'ts_channel_up_time':'daily trace segment length \n(seconds)', - 'ts_channel_continuity':'trace segment length (seconds) \n', - 'gain_ratio':'data/metadata gain ratio', # no units - 'phase_diff':'data-metadata phase difference \n(degrees)', - 'ms_coherence':'coherence function \n', # no units - } - + metricLabels = { + "amplifier_saturation": "daily flag count \n(number of occurrences)", + "calibration_signal": "daily flag count \n(number of occurrences)", + "clock_locked": "daily flag count \n(number of occurrences)", + "cross_talk": "correlation coefficient \n", # no units + "data_latency": "latency (seconds) \n", + "dc_offset": "daily indicator of likelihood of \nDC offset shift", # no units + "dead_channel_gsn": "indicator \n", + "dead_channel_lin": "standard deviation of residuals (dB) \n", + "digital_filter_charging": "daily flag count \n(number of occurrences)", + "digitizer_clipping": "daily flag count \n(number of occurrences)", + "event_begin": "daily flag count \n(number of occurrences)", + "event_end": "daily flag count \n(number of occurrences)", + "event_in_progress": "daily flag count \n(number of occurrences)", + "feed_latency": "latency (seconds) \n", + "gap_list": "daily gap length \n(seconds)", + "glitches": "daily flag count \n(number of occurrences)", + "max_gap": "daily maximum gap length \n(seconds)", + "max_overlap": "daily overlap length \n(seconds)", + "max_range": "daily maximum amplitude range, \nwindowed (counts)", + "max_stalta": "daily \nshort-term average / long-term \naverage", # no units + "missing_padded_data": "daily flag count \n(number of occurrences)", + "num_gaps": "daily gap count \n(number of occurrences)", + "num_overlaps": "daily overlap count \n(number of occurrences)", + "num_spikes": "daily outlier count \n(number of occurrences)", + "pct_above_nhnm": "daily PDF matrix above \nNew High Noise Model (%)", + "pct_below_nlnm": "daily PDF matrix below \nNew Low Noise Model (%)", + "percent_availability": "daily availability (%) \n", + "polarity_check": "maximum cross-correlation \nfunction", # no units + "pressure_effects": "daily zero-lag \ncross-correlation function", # no units + "sample_max": "daily maximum amplitude \n(counts)", + "sample_mean": "daily mean amplitude \n(counts)", + "sample_median": "daily median amplitude \n(counts)", + "sample_min": "daily minimum amplitude \n(counts)", + "sample_rate_channel": "daily indicator \n", + "sample_rate_resp": "daily indicator \n", + "sample_rms": "daily root-mean-square variance (counts) \n", + "scale_corrected_sample_rms": "daily root-mean-squared variance,\nscaled by sensitivity", + "sample_snr": "signal-to-noise ratio \n", # no units + "sample_unique": "daily unique sample values \n(number of occurrences)", + "spikes": "daily flag count \n(number of occurrences)", + "suspect_time_tag": "daily flag count \n(number of occurrences)", + "telemetry_sync_error": "daily flag count \n(number of occurrences)", + "timing_correction": "daily flag count \n(number of occurrences)", + "timing_quality": "daily average timing quality (%) \n", + "total_latency": "latency (seconds) \n", + "ts_num_gaps": "daily gap count \n(number of occurrences)", + "ts_num_gaps_total": "gap count \n(number of occurrences)", + "ts_max_gap": "daily maximum gap length \n(seconds)", + "ts_max_gap_total": "maximum gap length \n(seconds)", + "ts_gap_length": "daily total gap length \n(seconds)", + "ts_gap_length_total": "total gap length (seconds) \n", + "ts_percent_availability": "daily availability (%) \n", + "ts_percent_availability_total": "availability (%) \n", + "ts_channel_up_time": "daily trace segment length \n(seconds)", + "ts_channel_continuity": "trace segment length (seconds) \n", + "gain_ratio": "data/metadata gain ratio", # no units + "phase_diff": "data-metadata phase difference \n(degrees)", + "ms_coherence": "coherence function \n", # no units + } + labelText = metricLabels[metric] return labelText