#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""fdsnws_fetch
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
:Copyright:
2019-2024 Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences (Andres Heinloo)
:License:
LGPLv3 GNU Lesser General Public License v. 3 (29 June 2007, or later)
:Platform:
Linux
"""
"""
A command-line FDSN Web Service client using EIDA routing and authentication.
Usage Examples
==============
Request 60 minutes of the ``"LHZ"`` channel of EIDA stations starting with
``"A"`` for a seismic event around 2010-02-27 07:00 (UTC). Optionally add
``"-v"`` for verbosity. Resulting Mini-SEED data will be written to file
``"data.mseed"``.
.. code-block:: bash
$ %(prog)s -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -v -o data.mseed
The above request is anonymous and therefore restricted data will not be
included. To include restricted data, use a file containing a token obtained
from an EIDA authentication service and/or a CSV file with username and
password for each node not implementing the EIDA auth extension.
.. code-block:: bash
$ %(prog)s -a token.asc -c credentials.csv -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -v -o data.mseed
StationXML metadata for the above request can be requested using the following
command:
.. code-block:: bash
$ %(prog)s -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -y station \
-q level=response -v -o station.xml
Multiple query parameters can be used:
.. code-block:: bash
$ %(prog)s -N '*' -S '*' -L '*' -C '*' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -y station \
-q format=text -q level=channel -q latitude=20 -q longitude=-150 \
-q maxradius=15 -v -o station.txt
Bulk requests can be made in ArcLink (-f), breq_fast (-b) or native FDSNWS POST
(-p) format. Query parameters should not be included in the request file, but
specified on the command line.
.. code-block:: bash
$ %(prog)s -p request.txt -y station -q level=channel -v -o station.xml
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import sys
import time
import datetime
import optparse
import threading
import socket
import csv
import re
import struct
import io
import os
import fnmatch
import subprocess
import dateutil.parser
try:
# Python 3.2 and earlier
from xml.etree import cElementTree as ET # NOQA
except ImportError:
from xml.etree import ElementTree as ET # NOQA
try:
# Python 2.x
import Queue
import urllib2
import urlparse
import urllib
except ImportError:
# Python 3.x
import queue as Queue
import urllib.request as urllib2
import urllib.parse as urlparse
import urllib.parse as urllib
VERSION = "2022.017"
GET_PARAMS = set(('net', 'network',
'sta', 'station',
'loc', 'location',
'cha', 'channel',
'start', 'starttime',
'end', 'endtime',
'service',
'alternative'))
POST_PARAMS = set(('service',
'alternative'))
STATIONXML_RESOURCE_METADATA_ELEMENTS = (
'{http://www.fdsn.org/xml/station/1}Source',
'{http://www.fdsn.org/xml/station/1}Created',
'{http://www.fdsn.org/xml/station/1}Sender',
'{http://www.fdsn.org/xml/station/1}Module',
'{http://www.fdsn.org/xml/station/1}ModuleURI')
FIXED_DATA_HEADER_SIZE = 48
DATA_ONLY_BLOCKETTE_SIZE = 8
DATA_ONLY_BLOCKETTE_NUMBER = 1000
MINIMUM_RECORD_LENGTH = 256
DEFAULT_TOKEN_LOCATION = os.environ.get("HOME", "") + "/.eidatoken"
class Error(Exception):
pass
class AuthNotSupported(Exception):
pass
class TargetURL(object):
def __init__(self, url, qp):
self.__scheme = url.scheme
self.__netloc = url.netloc
self.__path = url.path.rstrip('query').rstrip('/')
self.__qp = dict(qp)
def wadl(self):
path = self.__path + '/application.wadl'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def auth(self):
path = self.__path + '/auth'
return urlparse.urlunparse(('https',
self.__netloc,
path,
'',
'',
''))
def post(self):
path = self.__path + '/query'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_qa(self):
path = self.__path + '/queryauth'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_params(self):
return self.__qp.items()
class RoutingURL(object):
def __init__(self, url, qp):
self.__scheme = url.scheme
self.__netloc = url.netloc
self.__path = url.path.rstrip('query').rstrip('/')
self.__qp = dict(qp)
def get(self):
path = self.__path + '/query'
qp = [(p, v) for (p, v) in self.__qp.items() if p in GET_PARAMS]
qp.append(('format', 'post'))
query = urllib.urlencode(qp)
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
query,
''))
def post(self):
path = self.__path + '/query'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_params(self):
qp = [(p, v) for (p, v) in self.__qp.items() if p in POST_PARAMS]
qp.append(('format', 'post'))
return qp
def target_params(self):
return [(p, v) for (p, v) in self.__qp.items() if p not in GET_PARAMS]
class TextCombiner(object):
def __init__(self):
self.__header = bytes()
self.__text = bytes()
def set_header(self, text):
self.__header = text
def combine(self, text):
self.__text += text
def dump(self, fd):
if self.__text:
fd.write(self.__header + self.__text)
class XMLCombiner(object):
def __init__(self):
self.__et = None
def __combine_element(self, one, other):
mapping = {}
for el in one:
try:
eid = (el.tag, el.attrib['code'], el.attrib['startDate'])
mapping[eid] = el
except KeyError:
pass
for el in other:
# skip Sender, Source, Module, ModuleURI, Created elements of
# subsequent trees
if el.tag in STATIONXML_RESOURCE_METADATA_ELEMENTS:
continue
try:
eid = (el.tag, el.attrib['code'], el.attrib['startDate'])
try:
self.__combine_element(mapping[eid], el)
except KeyError:
mapping[eid] = el
one.append(el)
except KeyError:
pass
def combine(self, et):
if self.__et:
self.__combine_element(self.__et.getroot(), et.getroot())
else:
self.__et = et
root = self.__et.getroot()
# Note: this assumes well-formed StationXML
# first StationXML tree: modify Source, Created
try:
source = root.find(STATIONXML_RESOURCE_METADATA_ELEMENTS[0])
source.text = 'FDSNWS'
except Exception:
pass
try:
created = root.find(STATIONXML_RESOURCE_METADATA_ELEMENTS[1])
created.text = datetime.datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%S')
except Exception:
pass
# remove Sender, Module, ModuleURI
for tag in STATIONXML_RESOURCE_METADATA_ELEMENTS[2:]:
el = root.find(tag)
if el is not None:
root.remove(el)
def dump(self, fd):
if self.__et:
self.__et.write(fd)
class ArclinkParser(object):
def __init__(self):
self.postdata = ""
self.failstr = ""
def __parse_line(self, line):
items = line.split()
if len(items) < 2:
self.failstr += "%s [syntax error]\n" % line
return
try:
beg_time = datetime.datetime(*map(int, items[0].split(",")))
end_time = datetime.datetime(*map(int, items[1].split(",")))
except ValueError as e:
self.failstr += "%s [invalid begin or end time: %s]\n" \
% (line, str(e))
return
network = 'XX'
station = 'XXXXX'
channel = 'XXX'
location = '--'
if len(items) > 2 and items[2] != '.':
network = items[2]
if len(items) > 3 and items[3] != '.':
station = items[3]
if len(items) > 4 and items[4] != '.':
channel = items[4]
if len(items) > 5 and items[5] != '.':
location = items[5]
self.postdata += "%s %s %s %s %sZ %sZ\n" \
% (network,
station,
location,
channel,
beg_time.isoformat(),
end_time.isoformat())
def parse(self, path):
with open(path) as fd:
for line in fd:
line = line.rstrip()
if line:
self.__parse_line(line)
class BreqParser(object):
__tokenrule = "^\.[A-Z_]+[:]?\s"
__reqlist = ("(?P[\w?\*]+)",
"(?P[\w?]+)",
"((?P\d{2})|(?P\d{4}))",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})(\.\d*)?",
"((?P\d{2})|(?P\d{4}))",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})",
"(?P\d{1,2})(\.\d*)?",
"(?P\d+)",
"(?P[\w?\s*]+)")
def __init__(self):
self.__rx_tokenrule = re.compile(BreqParser.__tokenrule)
self.__rx_reqlist = re.compile("\s+".join(BreqParser.__reqlist))
self.postdata = ""
self.failstr = ""
def __parse_line(self, line):
m = self.__rx_reqlist.match(line)
if m:
d = m.groupdict()
# catch two digit year inputs
if d["beg_2year"]:
if int(d["beg_2year"]) > 50:
d["beg_4year"] = "19%s" % d["beg_2year"]
else:
d["beg_4year"] = "20%s" % d["beg_2year"]
if d["end_2year"]:
if int(d["end_2year"]) > 50:
d["end_4year"] = "19%s" % d["end_2year"]
else:
d["end_4year"] = "20%s" % d["end_2year"]
# some users have problems with time...
if int(d["beg_hour"]) > 23:
d["beg_hour"] = "23"
if int(d["end_hour"]) > 23:
d["end_hour"] = "23"
if int(d["beg_min"]) > 59:
d["beg_min"] = "59"
if int(d["end_min"]) > 59:
d["end_min"] = "59"
if int(d["beg_sec"]) > 59:
d["beg_sec"] = "59"
if int(d["end_sec"]) > 59:
d["end_sec"] = "59"
try:
beg_time = datetime.datetime(int(d["beg_4year"]),
int(d["beg_month"]),
int(d["beg_day"]),
int(d["beg_hour"]),
int(d["beg_min"]),
int(d["beg_sec"]))
end_time = datetime.datetime(int(d["end_4year"]),
int(d["end_month"]),
int(d["end_day"]),
int(d["end_hour"]),
int(d["end_min"]),
int(d["end_sec"]))
except ValueError as e:
self.failstr += "%s [error: wrong begin or end time: %s]\n" \
% (line, str(e))
return
location = "*"
cha_list = re.findall("([\w?\*]+)\s*", d["cha_list"])
if len(cha_list) == int(d['cha_num'])+1:
location = cha_list.pop()
for channel in cha_list:
self.postdata += "%s %s %s %s %sZ %sZ\n" \
% (d["network"],
d["station"],
location,
channel,
beg_time.isoformat(),
end_time.isoformat())
else:
self.failstr += "%s [syntax error]\n" % line
def parse(self, path):
with open(path) as fd:
for line in fd:
if self.__rx_tokenrule.match(line):
continue
line = line.rstrip()
if line:
self.__parse_line(line)
msglock = threading.Lock()
def msg(s, verbose=3):
if verbose:
if verbose == 3:
if sys.stderr.isatty():
s = "\033[31m" + s + "\033[m"
elif verbose == 2:
if sys.stderr.isatty():
s = "\033[32m" + s + "\033[m"
with msglock:
sys.stderr.write(s + '\n')
sys.stderr.flush()
def retry(urlopen, url, data, timeout, count, wait, verbose):
# force no gzip encoding, because urllib cannot handle this
url = urllib2.Request(url, None, {"Accept-Encoding": ""})
n = 0
while True:
if n >= count:
return urlopen(url, data, timeout)
try:
n += 1
fd = urlopen(url, data, timeout)
if fd.getcode() == 200 or fd.getcode() == 204:
return fd
msg("retrying %s (%d) after %d seconds due to HTTP status code %d"
% (url, n, wait, fd.getcode()), verbose)
fd.close()
time.sleep(wait)
except urllib2.HTTPError as e:
if e.code >= 400 and e.code < 500:
raise
msg("retrying %s (%d) after %d seconds due to %s"
% (url, n, wait, str(e)), verbose)
time.sleep(wait)
except (urllib2.URLError, socket.error) as e:
msg("retrying %s (%d) after %d seconds due to %s"
% (url, n, wait, str(e)), verbose)
time.sleep(wait)
def fetch(url, cred, authdata, postlines, xc, tc, dest, nets, chans,
timeout, retry_count, retry_wait, finished, lock, verbose):
try:
url_handlers = []
if cred and url.post_qa() in cred: # use static credentials
query_url = url.post_qa()
(user, passwd) = cred[query_url]
mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, query_url, user, passwd)
h = urllib2.HTTPDigestAuthHandler(mgr)
url_handlers.append(h)
elif authdata: # use the pgp-based auth method if supported
wadl_url = url.wadl()
auth_url = url.auth()
query_url = url.post_qa()
try:
fd = retry(urllib2.urlopen, wadl_url, None, timeout,
retry_count, retry_wait, verbose)
try:
root = ET.parse(fd).getroot()
ns = "{http://wadl.dev.java.net/2009/02}"
el = "resource[@path='auth']"
if root.find(".//" + ns + el) is None:
raise AuthNotSupported
finally:
fd.close()
msg("authenticating at %s" % auth_url, verbose)
try:
fd = retry(urllib2.urlopen, auth_url, authdata, timeout,
retry_count, retry_wait, verbose)
try:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
if fd.getcode() == 200:
try:
(user, passwd) = resp.split(':')
mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, query_url, user, passwd)
h = urllib2.HTTPDigestAuthHandler(mgr)
url_handlers.append(h)
except ValueError:
msg("invalid auth response: %s" % resp)
return
msg("authentication at %s successful"
% auth_url, verbose)
else:
msg("authentication at %s failed with HTTP status "
"code %d:\n%s" % (auth_url, fd.getcode(), resp))
query_url = url.post()
finally:
fd.close()
except urllib2.HTTPError as e:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("authentication at %s failed with HTTP status "
"code %d:\n%s" % (auth_url, e.code, resp))
query_url = url.post()
except (urllib2.URLError, socket.error) as e:
msg("authentication at %s failed: %s" % (auth_url, str(e)))
query_url = url.post()
except (urllib2.URLError, socket.error, ET.ParseError) as e:
msg("reading %s failed: %s" % (wadl_url, str(e)))
query_url = url.post()
except AuthNotSupported:
msg("authentication at %s is not supported"
% auth_url, verbose)
query_url = url.post()
else: # fetch data anonymously
query_url = url.post()
opener = urllib2.build_opener(*url_handlers)
i = 0
n = len(postlines)
while i < len(postlines):
if n == len(postlines):
msg("getting data from %s" % query_url, verbose)
else:
msg("getting data from %s (%d%%..%d%%)"
% (query_url,
100*i/len(postlines),
min(100, 100*(i+n)/len(postlines))),
verbose)
postdata = (''.join((p + '=' + v + '\n')
for (p, v) in url.post_params()) +
''.join(postlines[i:i+n]))
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
fd = retry(opener.open, query_url, postdata, timeout,
retry_count, retry_wait, verbose)
try:
if fd.getcode() == 204:
msg("received no data from %s" % query_url)
elif fd.getcode() != 200:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting data from %s failed with HTTP status "
"code %d:\n%s" % (query_url, fd.getcode(), resp))
break
else:
size = 0
content_type = fd.info().get('Content-Type')
content_type = content_type.split(';')[0]
if content_type == "application/vnd.fdsn.mseed":
record_idx = 1
# NOTE: cannot use fixed chunk size, because
# response from single node mixes mseed record
# sizes. E.g., a 4096 byte chunk could contain 7
# 512 byte records and the first 512 bytes of a
# 4096 byte record, which would not be completed
# in the same write operation
while True:
# read fixed header
buf = fd.read(FIXED_DATA_HEADER_SIZE)
if not buf:
break
record = buf
curr_size = len(buf)
# get offset of data (value before last,
# 2 bytes, unsigned short)
data_offset_idx = FIXED_DATA_HEADER_SIZE - 4
data_offset, = struct.unpack(
b'!H',
buf[data_offset_idx:data_offset_idx+2])
if data_offset >= FIXED_DATA_HEADER_SIZE:
remaining_header_size = data_offset - \
FIXED_DATA_HEADER_SIZE
elif data_offset == 0 :
# This means that blockettes can follow,
# but no data samples. Use minimum record
# size to read following blockettes. This
# can still fail if blockette 1000 is after
# position 256
remaining_header_size = \
MINIMUM_RECORD_LENGTH - \
FIXED_DATA_HEADER_SIZE
else:
# Full header size cannot be smaller than
# fixed header size. This is an error.
msg("record %s: data offset smaller than "\
"fixed header length: %s, bailing "\
"out" % (record_idx, data_offset))
break
buf = fd.read(remaining_header_size)
if not buf:
msg("remaining header corrupt in record "\
"%s" % record_idx)
break
record += buf
curr_size += len(buf)
# scan variable header for blockette 1000
blockette_start = 0
b1000_found = False
while (blockette_start < remaining_header_size):
# 2 bytes, unsigned short
blockette_id, = struct.unpack(
b'!H',
buf[blockette_start:blockette_start+2])
# get start of next blockette (second
# value, 2 bytes, unsigned short)
next_blockette_start, = struct.unpack(
b'!H',
buf[blockette_start+2:blockette_start+4])
if blockette_id == \
DATA_ONLY_BLOCKETTE_NUMBER:
b1000_found = True
break
elif next_blockette_start == 0:
# no blockettes follow
msg("record %s: no blockettes follow "\
"after blockette %s at pos %s" % (
record_idx, blockette_id,
blockette_start))
break
else:
blockette_start = next_blockette_start - FIXED_DATA_HEADER_SIZE
# blockette 1000 not found
if not b1000_found:
msg("record %s: blockette 1000 not found,"\
" stop reading" % record_idx)
break
# get record size (1 byte, unsigned char)
record_size_exponent_idx = blockette_start + 6
record_size_exponent, = struct.unpack(
b'!B',
buf[record_size_exponent_idx:\
record_size_exponent_idx+1])
remaining_record_size = \
2**record_size_exponent - curr_size
# read remainder of record (data section)
buf = fd.read(remaining_record_size)
if not buf:
msg("cannot read data section of record "\
"%s" % record_idx)
break
record += buf
# collect network IDs
try:
net = record[18:20].decode('ascii').rstrip()
sta = record[8:13].decode('ascii').rstrip()
loc = record[13:15].decode('ascii').rstrip()
cha = record[15:18].decode('ascii').rstrip()
except UnicodeDecodeError:
msg("invalid miniseed record")
break
year, = struct.unpack(b'!H', record[20:22])
with lock:
nets.add((net, year))
chans.add('.'.join((net, sta, loc, cha)))
dest.write(record)
size += len(record)
record_idx += 1
elif content_type == "text/plain":
# this is the station service in text format
text = bytes()
while True:
buf = fd.readline()
if not buf:
break
if buf.startswith(b'#'):
tc.set_header(buf)
else:
text += buf
size += len(buf)
with lock:
tc.combine(text)
elif content_type == "application/xml":
fdread = fd.read
s = [0]
def read(self, *args, **kwargs):
buf = fdread(self, *args, **kwargs)
s[0] += len(buf)
return buf
fd.read = read
et = ET.parse(fd)
size = s[0]
with lock:
xc.combine(et)
else:
msg("getting data from %s failed: unsupported "
"content type '%s'" % (query_url,
content_type))
break
msg("got %d bytes (%s) from %s"
% (size, content_type, query_url), verbose)
i += n
finally:
fd.close()
except urllib2.HTTPError as e:
if e.code == 413 and n > 1:
msg("request too large for %s, splitting"
% query_url, verbose)
n = -(n//-2)
else:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting data from %s failed with HTTP status "
"code %d:\n%s" % (query_url, e.code, resp))
break
except (urllib2.URLError, socket.error, ET.ParseError) as e:
msg("getting data from %s failed: %s"
% (query_url, str(e)))
break
finally:
finished.put(threading.current_thread())
def route(url, cred, authdata, postdata, dest, chans_to_check, timeout,
retry_count, retry_wait, maxthreads, verbose):
threads = []
running = 0
finished = Queue.Queue()
lock = threading.Lock()
xc = XMLCombiner()
tc = TextCombiner()
nets = set()
check = bool(chans_to_check)
chans1 = chans_to_check
chans2 = set()
chans3 = set()
if postdata:
query_url = url.post()
postdata = (''.join((p + '=' + v + '\n')
for (p, v) in url.post_params()) +
postdata)
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
else:
query_url = url.get()
msg("getting routes from %s" % query_url, verbose)
try:
fd = retry(urllib2.urlopen, query_url, postdata, timeout, retry_count,
retry_wait, verbose)
try:
if fd.getcode() == 204:
msg("received no routes from %s" % query_url)
elif fd.getcode() != 200:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting routes from %s failed with HTTP status "
"code %d:\n%s" % (query_url, fd.getcode(), resp))
else:
urlline = None
postlines = []
while True:
line = fd.readline()
if isinstance(line, bytes):
line = line.decode('utf-8')
if not urlline:
urlline = line.strip()
elif not line.strip():
if postlines:
target_url = TargetURL(urlparse.urlparse(urlline),
url.target_params())
threads.append(threading.Thread(target=fetch,
args=(target_url,
cred,
authdata,
postlines,
xc,
tc,
dest,
nets,
chans3,
timeout,
retry_count,
retry_wait,
finished,
lock,
verbose)))
urlline = None
postlines = []
if not line:
break
else:
postlines.append(line)
if check:
nslc = line.split()[:4]
if nslc[2] == '--': nslc[2] = ''
chans2.add('.'.join(nslc))
finally:
fd.close()
except urllib2.HTTPError as e:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting routes from %s failed with HTTP status "
"code %d:\n%s" % (query_url, e.code, resp))
except (urllib2.URLError, socket.error) as e:
msg("getting routes from %s failed: %s" % (query_url, str(e)))
if check:
for c1 in list(chans1):
for c2 in list(chans2):
if fnmatch.fnmatch(c2, c1):
chans1.remove(c1)
break
if chans1:
msg("did not receive routes to %s" % ", ".join(sorted(chans1)))
for t in threads:
if running >= maxthreads:
thr = finished.get(True)
thr.join()
running -= 1
t.start()
running += 1
while running:
thr = finished.get(True)
thr.join()
running -= 1
xc.dump(dest)
tc.dump(dest)
if check:
for p in url.post_params():
if p[0] == 'service' and p[1] != 'dataselect':
return nets
for c2 in list(chans2):
for c3 in list(chans3):
if fnmatch.fnmatch(c3, c2):
chans2.remove(c2)
break
if chans2:
msg("did not receive data from %s" % ", ".join(sorted(chans2)))
return nets
def get_citation(nets, options):
postdata = ""
for (net, year) in nets:
postdata += "%s * * * %d-01-01T00:00:00Z %d-12-31T23:59:59Z\n" \
% (net, year, year)
qp = { 'service': 'station', 'level': 'network', 'format': 'text' }
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = io.BytesIO()
route(url, None, None, postdata, dest, None, options.timeout,
options.retries, options.retry_wait, options.threads,
options.verbose)
dest.seek(0)
net_desc = {}
for line in dest:
try:
if isinstance(line, bytes):
line = line.decode('utf-8')
(code, desc, start) = line.split('|')[:3]
if code.startswith('#'):
continue
year = dateutil.parser.parse(start).year
except (ValueError, UnicodeDecodeError) as e:
msg("error parsing text format: %s" % str(e))
continue
if code[0] in '0123456789XYZ':
net_desc['%s_%d' % (code, year)] = desc
else:
net_desc[code] = desc
msg("\nYou received seismic waveform data from the following network(s):", 2)
for code in sorted(net_desc):
msg("%s %s" % (code, net_desc[code]), 2)
msg("\nAcknowledgment is extremely important for network operators\n"
"providing open data. When preparing publications, please\n"
"cite the data appropriately. The FDSN service at\n\n"
" http://www.fdsn.org/networks/citation/?networks=%s\n\n"
"provides a helpful guide based on available network\n"
"Digital Object Identifiers.\n"
% "+".join(sorted(net_desc)), 2)
def main():
qp = {}
def add_qp(option, opt_str, value, parser):
if option.dest == 'query':
try:
(p, v) = value.split('=', 1)
qp[p] = v
except ValueError:
raise optparse.OptionValueError("%s expects parameter=value"
% opt_str)
else:
qp[option.dest] = value
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION,
add_help_option=False)
parser.set_defaults(
url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5)
parser.add_option("-h", "--help", action="store_true", default=False,
help="show help message and exit")
parser.add_option("-l", "--longhelp", action="store_true", default=False,
help="show extended help message and exit")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="verbose mode")
parser.add_option("-u", "--url", type="string",
help="URL of routing service (default %default)")
parser.add_option("-y", "--service", type="string", action="callback",
callback=add_qp,
help="target service (default dataselect)")
parser.add_option("-N", "--network", type="string", action="callback",
callback=add_qp,
help="network code or pattern")
parser.add_option("-S", "--station", type="string", action="callback",
callback=add_qp,
help="station code or pattern")
parser.add_option("-L", "--location", type="string", action="callback",
callback=add_qp,
help="location code or pattern")
parser.add_option("-C", "--channel", type="string", action="callback",
callback=add_qp,
help="channel code or pattern")
parser.add_option("-s", "--starttime", type="string", action="callback",
callback=add_qp,
help="start time")
parser.add_option("-e", "--endtime", type="string", action="callback",
callback=add_qp,
help="end time")
parser.add_option("-q", "--query", type="string", action="callback",
callback=add_qp, metavar="PARAMETER=VALUE",
help="additional query parameter")
parser.add_option("-t", "--timeout", type="int",
help="request timeout in seconds (default %default)")
parser.add_option("-r", "--retries", type="int",
help="number of retries (default %default)")
parser.add_option("-w", "--retry-wait", type="int",
help="seconds to wait before each retry "
"(default %default)")
parser.add_option("-n", "--threads", type="int",
help="maximum number of download threads "
"(default %default)")
parser.add_option("-c", "--credentials-file", type="string",
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a", "--auth-file", type="string",
help="file that contains the auth token")
parser.add_option("-p", "--post-file", type="string",
help="request file in FDSNWS POST format")
parser.add_option("-f", "--arclink-file", type="string",
help="request file in ArcLink format")
parser.add_option("-b", "--breqfast-file", type="string",
help="request file in breq_fast format")
parser.add_option("-o", "--output-file", type="string",
help="file where downloaded data is written")
parser.add_option("-z", "--no-citation", action="store_true", default=False,
help="suppress network citation info")
parser.add_option("-Z", "--no-check", action="store_true", default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if options.help:
print(__doc__.split("Usage Examples", 1)[0], end="")
parser.print_help()
return 0
if options.longhelp:
print(__doc__)
parser.print_help()
return 0
if args or not options.output_file:
parser.print_usage(sys.stderr)
return 1
if bool(options.post_file) + bool(options.arclink_file) + \
bool(options.breqfast_file) > 1:
msg("only one of (--post-file, --arclink-file, --breqfast-file) "
"can be used")
return 1
try:
cred = {}
authdata = None
postdata = None
chans_to_check = set()
if options.credentials_file:
with open(options.credentials_file) as fd:
try:
for (url, user, passwd) in csv.reader(fd):
cred[url] = (user, passwd)
except (ValueError, csv.Error):
raise Error("error parsing %s" % options.credentials_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.credentials_file)
if options.auth_file:
with open(options.auth_file, 'rb') as fd:
authdata = fd.read()
else:
try:
with open(DEFAULT_TOKEN_LOCATION, 'rb') as fd:
authdata = fd.read()
options.auth_file = DEFAULT_TOKEN_LOCATION
except IOError:
pass
if authdata:
msg("using token in %s:" % options.auth_file, options.verbose)
try:
proc = subprocess.Popen(['gpg', '--decrypt'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(authdata)
if not out:
if isinstance(err, bytes):
err = err.decode('utf-8')
msg(err)
return 1
if isinstance(out, bytes):
out = out.decode('utf-8')
msg(out, options.verbose)
except OSError as e:
msg(str(e))
if options.post_file:
try:
with open(options.post_file) as fd:
postdata = fd.read()
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.post_file)
else:
parser = None
if options.arclink_file:
parser = ArclinkParser()
try:
parser.parse(options.arclink_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.arclink_file)
elif options.breqfast_file:
parser = BreqParser()
try:
parser.parse(options.breqfast_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.breqfast_file)
if parser is not None:
if parser.failstr:
msg(parser.failstr)
return 1
postdata = parser.postdata
if not options.no_check:
if postdata:
for line in postdata.splitlines():
nslc = line.split()[:4]
if nslc[2] == '--': nslc[2] = ''
chans_to_check.add('.'.join(nslc))
else:
net = qp.get('network', '*')
sta = qp.get('station', '*')
loc = qp.get('location', '*')
cha = qp.get('channel', '*')
for n in net.split(','):
for s in sta.split(','):
for l in loc.split(','):
for c in cha.split(','):
if l == '--': l = ''
chans_to_check.add('.'.join((n, s, l, c)))
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = open(options.output_file, 'wb')
nets = route(url, cred, authdata, postdata, dest, chans_to_check,
options.timeout, options.retries, options.retry_wait,
options.threads, options.verbose)
if nets and not options.no_citation:
msg("retrieving network citation info", options.verbose)
get_citation(nets, options)
else:
msg("", options.verbose)
msg("In case of problems with your request, plese use the contact "
"form at\n\n"
" http://www.orfeus-eu.org/organization/contact/form/"
"?recipient=EIDA\n", options.verbose)
except (IOError, Error) as e:
msg(str(e))
return 1
return 0
if __name__ == "__main__":
__doc__ %= {"prog": sys.argv[0]}
sys.exit(main())