#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""fdsnws_fetch - A command-line FDSN Web Service client using EIDA routing
and authentication.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
:Copyright:
2019-2025 GFZ Helmholtz Centre for Geosciences
:License:
LGPLv3 GNU Lesser General Public License v. 3 (29 June 2007, or later)
:Platform:
Linux
Usage Examples
==============
Request 60 minutes of the ``"LHZ"`` channel of EIDA stations starting with
``"A"`` for a seismic event around 2010-02-27 07:00 (UTC). Optionally add
``"-v"`` for verbosity. Resulting Mini-SEED data will be written to file
``"data.mseed"``.
.. code-block:: bash
$ %(prog)s -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -v -o data.mseed
The above request is anonymous and therefore restricted data will not be
included. To include restricted data, use a file containing a token obtained
from an EIDA authentication service and/or a CSV file with username and
password for each node not implementing the EIDA auth extension.
.. code-block:: bash
$ %(prog)s -a token.asc -c credentials.csv -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -v -o data.mseed
StationXML metadata for the above request can be requested using the following
command:
.. code-block:: bash
$ %(prog)s -N '*' -S 'A*' -L '*' -C 'LHZ' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -y station \
-q level=response -v -o station.xml
Multiple query parameters can be used:
.. code-block:: bash
$ %(prog)s -N '*' -S '*' -L '*' -C '*' \
-s "2010-02-27T07:00:00Z" -e "2010-02-27T08:00:00Z" -y station \
-q format=text -q level=channel -q latitude=20 -q longitude=-150 \
-q maxradius=15 -v -o station.txt
Bulk requests can be made in ArcLink (-f), breq_fast (-b) or native FDSNWS POST
(-p) format. Query parameters should not be included in the request file, but
specified on the command line.
.. code-block:: bash
$ %(prog)s -p request.txt -y station -q level=channel -v -o station.xml
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import sys
import time
import datetime
import optparse
import threading
import socket
import csv
import re
import struct
import io
import os
import fnmatch
import subprocess
import dateutil.parser
import gzip
try:
# Python 3.2 and earlier
from xml.etree import cElementTree as ET # NOQA
except ImportError:
from xml.etree import ElementTree as ET # NOQA
try:
# Python 2.x
import Queue
import urllib2
import urlparse
import urllib
except ImportError:
# Python 3.x
import queue as Queue
import urllib.request as urllib2
import urllib.parse as urlparse
import urllib.parse as urllib
try:
import eas2cli.core
_jwt_supported = True
except ImportError:
print("Package eas2cli not found or not usable -- JWT support disabled",
file=sys.stderr)
_jwt_supported = False
VERSION = "2025.284"
GET_PARAMS = set(('net', 'network',
'sta', 'station',
'loc', 'location',
'cha', 'channel',
'start', 'starttime',
'end', 'endtime',
'service',
'alternative'))
POST_PARAMS = set(('service',
'alternative'))
STATIONXML_RESOURCE_METADATA_ELEMENTS = (
'{http://www.fdsn.org/xml/station/1}Source',
'{http://www.fdsn.org/xml/station/1}Created',
'{http://www.fdsn.org/xml/station/1}Sender',
'{http://www.fdsn.org/xml/station/1}Module',
'{http://www.fdsn.org/xml/station/1}ModuleURI')
FIXED_DATA_HEADER_SIZE = 48
DATA_ONLY_BLOCKETTE_SIZE = 8
DATA_ONLY_BLOCKETTE_NUMBER = 1000
MINIMUM_RECORD_LENGTH = 256
DEFAULT_TOKEN_LOCATION = os.environ.get("HOME", "") + "/.eidatoken"
DEFAULT_JWT_LOCATION = os.environ.get("HOME", "") + "/.eidajwt"
class Error(Exception):
pass
class AuthNotSupported(Exception):
pass
class TargetURL(object):
def __init__(self, url, qp):
self.__scheme = url.scheme
self.__netloc = url.netloc
self.__path = url.path.rstrip('query').rstrip('/')
self.__qp = dict(qp)
def wadl(self):
path = self.__path + '/application.wadl'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def auth(self):
path = self.__path + '/auth'
return urlparse.urlunparse(('https',
self.__netloc,
path,
'',
'',
''))
def post(self):
path = self.__path + '/query'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_qa(self):
path = self.__path + '/queryauth'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_params(self):
return self.__qp.items()
class RoutingURL(object):
def __init__(self, url, qp):
self.__scheme = url.scheme
self.__netloc = url.netloc
self.__path = url.path.rstrip('query').rstrip('/')
self.__qp = dict(qp)
def get(self):
path = self.__path + '/query'
qp = [(p, v) for (p, v) in self.__qp.items() if p in GET_PARAMS]
qp.append(('format', 'post'))
query = urllib.urlencode(qp)
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
query,
''))
def post(self):
path = self.__path + '/query'
return urlparse.urlunparse((self.__scheme,
self.__netloc,
path,
'',
'',
''))
def post_params(self):
qp = [(p, v) for (p, v) in self.__qp.items() if p in POST_PARAMS]
qp.append(('format', 'post'))
return qp
def target_params(self):
return [(p, v) for (p, v) in self.__qp.items() if p not in GET_PARAMS]
class TextCombiner(object):
def __init__(self):
self.__header = bytes()
self.__text = bytes()
def set_header(self, text):
self.__header = text
def combine(self, text):
self.__text += text
def dump(self, fd):
if self.__text:
fd.write(self.__header + self.__text)
class XMLCombiner(object):
def __init__(self):
self.__et = None
def __combine_element(self, one, other):
mapping = {}
for el in one:
try:
eid = (el.tag, el.attrib['code'], el.attrib['startDate'])
mapping[eid] = el
except KeyError:
pass
for el in other:
# skip Sender, Source, Module, ModuleURI, Created elements of
# subsequent trees
if el.tag in STATIONXML_RESOURCE_METADATA_ELEMENTS:
continue
try:
eid = (el.tag, el.attrib['code'], el.attrib['startDate'])
try:
self.__combine_element(mapping[eid], el)
except KeyError:
mapping[eid] = el
one.append(el)
except KeyError:
pass
def combine(self, et):
if self.__et:
self.__combine_element(self.__et.getroot(), et.getroot())
else:
self.__et = et
root = self.__et.getroot()
# Note: this assumes well-formed StationXML
# first StationXML tree: modify Source, Created
try:
source = root.find(STATIONXML_RESOURCE_METADATA_ELEMENTS[0])
source.text = 'FDSNWS'
except Exception:
pass
try:
created = root.find(STATIONXML_RESOURCE_METADATA_ELEMENTS[1])
created.text = datetime.datetime.now(datetime.UTC).strftime(
'%Y-%m-%dT%H:%M:%S')
except Exception:
pass
# remove Sender, Module, ModuleURI
for tag in STATIONXML_RESOURCE_METADATA_ELEMENTS[2:]:
el = root.find(tag)
if el is not None:
root.remove(el)
def dump(self, fd):
if self.__et:
self.__et.write(fd)
class ArclinkParser(object):
def __init__(self):
self.postdata = ""
self.failstr = ""
def __parse_line(self, line):
items = line.split()
if len(items) < 2:
self.failstr += "%s [syntax error]\n" % line
return
try:
beg_time = datetime.datetime(*map(int, items[0].split(",")))
end_time = datetime.datetime(*map(int, items[1].split(",")))
except ValueError as e:
self.failstr += "%s [invalid begin or end time: %s]\n" \
% (line, str(e))
return
network = 'XX'
station = 'XXXXX'
channel = 'XXX'
location = '--'
if len(items) > 2 and items[2] != '.':
network = items[2]
if len(items) > 3 and items[3] != '.':
station = items[3]
if len(items) > 4 and items[4] != '.':
channel = items[4]
if len(items) > 5 and items[5] != '.':
location = items[5]
self.postdata += "%s %s %s %s %sZ %sZ\n" \
% (network,
station,
location,
channel,
beg_time.isoformat(),
end_time.isoformat())
def parse(self, path):
with open(path) as fd:
for line in fd:
line = line.rstrip()
if line:
self.__parse_line(line)
class BreqParser(object):
__tokenrule = r"^\.[A-Z_]+[:]?\s"
__reqlist = (r"(?P[\w?\*]+)",
r"(?P[\w?]+)",
r"((?P\d{2})|(?P\d{4}))",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})(\.\d*)?",
r"((?P\d{2})|(?P\d{4}))",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})",
r"(?P\d{1,2})(\.\d*)?",
r"(?P\d+)",
r"(?P[\w?\s*]+)")
def __init__(self):
self.__rx_tokenrule = re.compile(BreqParser.__tokenrule)
self.__rx_reqlist = re.compile(r"\s+".join(BreqParser.__reqlist))
self.postdata = ""
self.failstr = ""
def __parse_line(self, line):
m = self.__rx_reqlist.match(line)
if m:
d = m.groupdict()
# catch two digit year inputs
if d["beg_2year"]:
if int(d["beg_2year"]) > 50:
d["beg_4year"] = "19%s" % d["beg_2year"]
else:
d["beg_4year"] = "20%s" % d["beg_2year"]
if d["end_2year"]:
if int(d["end_2year"]) > 50:
d["end_4year"] = "19%s" % d["end_2year"]
else:
d["end_4year"] = "20%s" % d["end_2year"]
# some users have problems with time...
if int(d["beg_hour"]) > 23:
d["beg_hour"] = "23"
if int(d["end_hour"]) > 23:
d["end_hour"] = "23"
if int(d["beg_min"]) > 59:
d["beg_min"] = "59"
if int(d["end_min"]) > 59:
d["end_min"] = "59"
if int(d["beg_sec"]) > 59:
d["beg_sec"] = "59"
if int(d["end_sec"]) > 59:
d["end_sec"] = "59"
try:
beg_time = datetime.datetime(int(d["beg_4year"]),
int(d["beg_month"]),
int(d["beg_day"]),
int(d["beg_hour"]),
int(d["beg_min"]),
int(d["beg_sec"]))
end_time = datetime.datetime(int(d["end_4year"]),
int(d["end_month"]),
int(d["end_day"]),
int(d["end_hour"]),
int(d["end_min"]),
int(d["end_sec"]))
except ValueError as e:
self.failstr += "%s [error: wrong begin or end time: %s]\n" \
% (line, str(e))
return
location = "*"
cha_list = re.findall(r"([\w?\*]+)\s*", d["cha_list"])
if len(cha_list) == int(d['cha_num'])+1:
location = cha_list.pop()
for channel in cha_list:
self.postdata += "%s %s %s %s %sZ %sZ\n" \
% (d["network"],
d["station"],
location,
channel,
beg_time.isoformat(),
end_time.isoformat())
else:
self.failstr += "%s [syntax error]\n" % line
def parse(self, path):
with open(path) as fd:
for line in fd:
if self.__rx_tokenrule.match(line):
continue
line = line.rstrip()
if line:
self.__parse_line(line)
class JWTHandler(urllib2.BaseHandler):
errrx = re.compile(r"""(?: |,)error_description=(["'])((?:\\1|(?:(?!\1)).)*)(\1)""", re.I)
def __init__(self, jwt_file, verbose):
self.__jwt_file = jwt_file
self.__tokens = eas2cli.core.readtokens(jwt_file)
self.__refreshed = False
self.__verbose = verbose
def __refresh(self):
eas2cli.core._silentrefresh(reftok=self.__tokens.refresh_token, tokenfile=self.__jwt_file)
self.__tokens = eas2cli.core.readtokens(self.__jwt_file)
self.__refreshed = True
def https_request(self, req):
req.add_unredirected_header("Authorization", "Bearer " + self.__tokens.access_token)
return req
def https_response(self, req, response):
if response.code == 401:
if self.__refreshed: # avoid endless auth loop
fp = response
# replace response body by error description if available
headers = response.info()
authreq = headers.get_all("www-authenticate")
if authreq:
for hdr in authreq:
scheme = hdr.split()[0]
if scheme.lower() == "bearer":
m = JWTHandler.errrx.search(hdr)
if m:
err = m.group(2)
if err:
fp = io.StringIO("JWT error: " + err)
break
raise urllib2.HTTPError(req.full_url, response.code, response.msg,
headers, fp)
msg("refreshing token", self.__verbose)
try:
self.__refresh()
return self.parent.open(req, timeout=req.timeout)
except Exception as e:
msg(str(e))
# reset flag
self.__refreshed = False
return response
msglock = threading.Lock()
def msg(s, verbose=3):
if verbose:
if verbose == 3:
if sys.stderr.isatty():
s = "\033[31m" + s + "\033[m"
elif verbose == 2:
if sys.stderr.isatty():
s = "\033[32m" + s + "\033[m"
with msglock:
sys.stderr.write(s + '\n')
sys.stderr.flush()
def retry(urlopen, url, data, timeout, count, wait, verbose):
headers = {
"Accept-Encoding": "gzip",
"User-Agent": "fdsnws_fetch/" + VERSION
}
if data:
headers["Content-Type"] = "text/plain"
req = urllib2.Request(url, headers=headers)
n = 0
while True:
try:
n += 1
fd = urlopen(req, data, timeout)
code = fd.getcode()
info = fd.info()
if info.get("Content-Encoding") == "gzip":
fd = gzip.GzipFile(fileobj=fd, mode='rb')
fd.getcode = lambda: code
fd.info = lambda: info
if n > count:
return fd
if fd.getcode() == 200 or fd.getcode() == 204:
return fd
msg("retrying %s (%d) after %d seconds due to HTTP status code %d"
% (url, n, wait, fd.getcode()), verbose)
fd.close()
time.sleep(wait)
except urllib2.HTTPError as e:
if e.code >= 400 and e.code < 500:
if e.info().get("Content-Encoding") == "gzip":
with gzip.GzipFile(fileobj=e, mode='rb') as fd:
data = fd.read()
e.read = lambda: data
raise
msg("retrying %s (%d) after %d seconds due to %s"
% (url, n, wait, str(e)), verbose)
time.sleep(wait)
except (urllib2.URLError, socket.error) as e:
msg("retrying %s (%d) after %d seconds due to %s"
% (url, n, wait, str(e)), verbose)
time.sleep(wait)
def fetch(url, cred, authdata, jwt_file, postlines, xc, tc, dest, nets, chans,
timeout, retry_count, retry_wait, finished, lock, maxlines, verbose):
try:
url_handlers = []
if cred and url.post_qa() in cred: # use static credentials
query_url = url.post_qa()
(user, passwd) = cred[query_url]
mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, query_url, user, passwd)
h = urllib2.HTTPDigestAuthHandler(mgr)
url_handlers.append(h)
elif authdata: # use the pgp-based auth method if supported
wadl_url = url.wadl()
auth_url = url.auth()
query_url = url.post_qa()
try:
fd = retry(urllib2.urlopen, wadl_url, None, timeout,
retry_count, retry_wait, verbose)
try:
root = ET.parse(fd).getroot()
ns = "{http://wadl.dev.java.net/2009/02}"
el = "resource[@path='auth']"
if root.find(".//" + ns + el) is None:
raise AuthNotSupported
finally:
fd.close()
msg("authenticating at %s" % auth_url, verbose)
try:
fd = retry(urllib2.urlopen, auth_url, authdata, timeout,
retry_count, retry_wait, verbose)
try:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
if fd.getcode() == 200:
try:
(user, passwd) = resp.split(':')
mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, query_url, user, passwd)
h = urllib2.HTTPDigestAuthHandler(mgr)
url_handlers.append(h)
except ValueError:
msg("invalid auth response: %s" % resp)
return
msg("authentication at %s successful"
% auth_url, verbose)
else:
msg("authentication at %s failed with HTTP status "
"code %d:\n%s" % (auth_url, fd.getcode(), resp))
query_url = url.post()
finally:
fd.close()
except urllib2.HTTPError as e:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("authentication at %s failed with HTTP status "
"code %d:\n%s" % (auth_url, e.code, resp))
query_url = url.post()
except (urllib2.URLError, socket.error) as e:
msg("authentication at %s failed: %s" % (auth_url, str(e)))
query_url = url.post()
except (urllib2.URLError, socket.error, ET.ParseError) as e:
msg("reading %s failed: %s" % (wadl_url, str(e)))
query_url = url.post()
except AuthNotSupported:
msg("authentication at %s is not supported"
% auth_url, verbose)
query_url = url.post()
elif _jwt_supported and jwt_file: # use the JWT auth if supported
try:
url_handlers.append(JWTHandler(jwt_file, verbose))
except Exception as e:
msg(str(e))
query_url = url.post()
else: # fetch data anonymously
query_url = url.post()
opener = urllib2.build_opener(*url_handlers)
i = 0
n = min(len(postlines), maxlines)
while i < len(postlines):
if n == len(postlines):
msg("getting data from %s" % query_url, verbose)
else:
msg("getting data from %s (%d%%..%d%%)"
% (query_url,
100*i/len(postlines),
min(100, 100*(i+n)/len(postlines))),
verbose)
postdata = (''.join((p + '=' + v + '\n')
for (p, v) in url.post_params()) +
''.join(postlines[i:i+n]))
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
fd = retry(opener.open, query_url, postdata, timeout,
retry_count, retry_wait, verbose)
try:
if fd.getcode() == 204:
msg("received no data from %s" % query_url)
elif fd.getcode() != 200:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting data from %s failed with HTTP status "
"code %d:\n%s" % (query_url, fd.getcode(), resp))
break
else:
size = 0
content_type = fd.info().get('Content-Type')
content_type = content_type.split(';')[0]
if content_type == "application/vnd.fdsn.mseed":
record_idx = 1
# NOTE: cannot use fixed chunk size, because
# response from single node mixes mseed record
# sizes. E.g., a 4096 byte chunk could contain 7
# 512 byte records and the first 512 bytes of a
# 4096 byte record, which would not be completed
# in the same write operation
while True:
# read fixed header
buf = fd.read(FIXED_DATA_HEADER_SIZE)
if not buf:
break
record = buf
curr_size = len(buf)
# get offset of data (value before last,
# 2 bytes, unsigned short)
data_offset_idx = FIXED_DATA_HEADER_SIZE - 4
data_offset, = struct.unpack(
b'!H',
buf[data_offset_idx:data_offset_idx+2])
if data_offset >= FIXED_DATA_HEADER_SIZE:
remaining_header_size = data_offset - \
FIXED_DATA_HEADER_SIZE
elif data_offset == 0 :
# This means that blockettes can follow,
# but no data samples. Use minimum record
# size to read following blockettes. This
# can still fail if blockette 1000 is after
# position 256
remaining_header_size = \
MINIMUM_RECORD_LENGTH - \
FIXED_DATA_HEADER_SIZE
else:
# Full header size cannot be smaller than
# fixed header size. This is an error.
msg("record %s: data offset smaller than "\
"fixed header length: %s, bailing "\
"out" % (record_idx, data_offset))
break
buf = fd.read(remaining_header_size)
if not buf:
msg("remaining header corrupt in record "\
"%s" % record_idx)
break
record += buf
curr_size += len(buf)
# scan variable header for blockette 1000
blockette_start = 0
b1000_found = False
while (blockette_start < remaining_header_size):
# 2 bytes, unsigned short
blockette_id, = struct.unpack(
b'!H',
buf[blockette_start:blockette_start+2])
# get start of next blockette (second
# value, 2 bytes, unsigned short)
next_blockette_start, = struct.unpack(
b'!H',
buf[blockette_start+2:blockette_start+4])
if blockette_id == \
DATA_ONLY_BLOCKETTE_NUMBER:
b1000_found = True
break
elif next_blockette_start == 0:
# no blockettes follow
msg("record %s: no blockettes follow "\
"after blockette %s at pos %s" % (
record_idx, blockette_id,
blockette_start))
break
else:
blockette_start = next_blockette_start - FIXED_DATA_HEADER_SIZE
# blockette 1000 not found
if not b1000_found:
msg("record %s: blockette 1000 not found,"\
" stop reading" % record_idx)
break
# get record size (1 byte, unsigned char)
record_size_exponent_idx = blockette_start + 6
record_size_exponent, = struct.unpack(
b'!B',
buf[record_size_exponent_idx:\
record_size_exponent_idx+1])
remaining_record_size = \
2**record_size_exponent - curr_size
# read remainder of record (data section)
buf = fd.read(remaining_record_size)
if not buf:
msg("cannot read data section of record "\
"%s" % record_idx)
break
record += buf
# collect network IDs
try:
net = record[18:20].decode('ascii').rstrip()
sta = record[8:13].decode('ascii').rstrip()
loc = record[13:15].decode('ascii').rstrip()
cha = record[15:18].decode('ascii').rstrip()
except UnicodeDecodeError:
msg("invalid miniseed record")
break
year, = struct.unpack(b'!H', record[20:22])
with lock:
nets.add((net, year))
chans.add('.'.join((net, sta, loc, cha)))
dest.write(record)
size += len(record)
record_idx += 1
elif content_type == "text/plain":
# this is the station service in text format
text = bytes()
while True:
buf = fd.readline()
if not buf:
break
if buf.startswith(b'#'):
tc.set_header(buf)
else:
text += buf
size += len(buf)
with lock:
tc.combine(text)
elif content_type == "application/xml":
fdread = fd.read
s = [0]
def read(self, *args, **kwargs):
buf = fdread(self, *args, **kwargs)
s[0] += len(buf)
return buf
fd.read = read
et = ET.parse(fd)
size = s[0]
with lock:
xc.combine(et)
else:
msg("getting data from %s failed: unsupported "
"content type '%s'" % (query_url,
content_type))
break
msg("got %d bytes (%s) from %s"
% (size, content_type, query_url), verbose)
i += n
finally:
fd.close()
except urllib2.HTTPError as e:
if e.code == 413 and n > 1:
msg("request too large for %s, splitting"
% query_url, verbose)
n = -(n//-2)
else:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting data from %s failed with HTTP status "
"code %d:\n%s" % (query_url, e.code, resp))
break
except (urllib2.URLError, socket.error, ET.ParseError) as e:
msg("getting data from %s failed: %s"
% (query_url, str(e)))
break
finally:
finished.put(threading.current_thread())
def route(url, cred, authdata, jwt_file, postdata, dest, chans_to_check, timeout,
retry_count, retry_wait, maxthreads, maxlines, verbose):
threads = []
running = 0
finished = Queue.Queue()
lock = threading.Lock()
xc = XMLCombiner()
tc = TextCombiner()
nets = set()
check = bool(chans_to_check)
chans1 = chans_to_check
chans2 = set()
chans3 = set()
if postdata:
query_url = url.post()
postdata = (''.join((p + '=' + v + '\n')
for (p, v) in url.post_params()) +
postdata)
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
else:
query_url = url.get()
msg("getting routes from %s" % query_url, verbose)
try:
fd = retry(urllib2.urlopen, query_url, postdata, timeout, retry_count,
retry_wait, verbose)
try:
if fd.getcode() == 204:
msg("received no routes from %s" % query_url)
elif fd.getcode() != 200:
resp = fd.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting routes from %s failed with HTTP status "
"code %d:\n%s" % (query_url, fd.getcode(), resp))
else:
urlline = None
postlines = []
while True:
line = fd.readline()
if isinstance(line, bytes):
line = line.decode('utf-8')
if not urlline:
urlline = line.strip()
elif not line.strip():
if postlines:
target_url = TargetURL(urlparse.urlparse(urlline),
url.target_params())
threads.append(threading.Thread(target=fetch,
args=(target_url,
cred,
authdata,
jwt_file,
postlines,
xc,
tc,
dest,
nets,
chans3,
timeout,
retry_count,
retry_wait,
finished,
lock,
maxlines,
verbose)))
urlline = None
postlines = []
if not line:
break
else:
postlines.append(line)
if check:
nslc = line.split()[:4]
if nslc[2] == '--': nslc[2] = ''
chans2.add('.'.join(nslc))
finally:
fd.close()
except urllib2.HTTPError as e:
resp = e.read()
if isinstance(resp, bytes):
resp = resp.decode('utf-8')
msg("getting routes from %s failed with HTTP status "
"code %d:\n%s" % (query_url, e.code, resp))
except (urllib2.URLError, socket.error) as e:
msg("getting routes from %s failed: %s" % (query_url, str(e)))
if check:
for c1 in list(chans1):
for c2 in list(chans2):
if fnmatch.fnmatch(c2, c1):
chans1.remove(c1)
break
if chans1:
msg("did not receive routes to %s" % ", ".join(sorted(chans1)))
for t in threads:
if running >= maxthreads:
thr = finished.get(True)
thr.join()
running -= 1
t.start()
running += 1
while running:
thr = finished.get(True)
thr.join()
running -= 1
xc.dump(dest)
tc.dump(dest)
if check:
for p in url.post_params():
if p[0] == 'service' and p[1] != 'dataselect':
return nets
for c2 in list(chans2):
for c3 in list(chans3):
if fnmatch.fnmatch(c3, c2):
chans2.remove(c2)
break
if chans2:
msg("did not receive data from %s" % ", ".join(sorted(chans2)))
return nets
def get_citation(nets, options):
postdata = ""
for (net, year) in nets:
postdata += "%s * * * %d-01-01T00:00:00Z %d-12-31T23:59:59Z\n" \
% (net, year, year)
qp = { 'service': 'station', 'level': 'network', 'format': 'text' }
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = io.BytesIO()
route(url, None, None, None, postdata, dest, None, options.timeout,
options.retries, options.retry_wait, options.threads,
options.max_lines, options.verbose)
dest.seek(0)
net_desc = {}
for line in dest:
try:
if isinstance(line, bytes):
line = line.decode('utf-8')
(code, desc, start) = line.split('|')[:3]
if code.startswith('#'):
continue
year = dateutil.parser.parse(start).year
except (ValueError, UnicodeDecodeError) as e:
msg("error parsing text format: %s" % str(e))
continue
if code[0] in '0123456789XYZ':
net_desc['%s_%d' % (code, year)] = desc
else:
net_desc[code] = desc
msg("\nYou received seismic waveform data from the following network(s):", 2)
for code in sorted(net_desc):
msg("%s %s" % (code, net_desc[code]), 2)
msg("\nAcknowledgment is extremely important for network operators\n"
"providing open data. When preparing publications, please\n"
"cite the data appropriately. The FDSN service at\n\n"
" http://www.fdsn.org/networks/citation/?networks=%s\n\n"
"provides a helpful guide based on available network\n"
"Digital Object Identifiers.\n"
% "+".join(sorted(net_desc)), 2)
def main():
qp = {}
def add_qp(option, opt_str, value, parser):
if option.dest == 'query':
try:
(p, v) = value.split('=', 1)
qp[p] = v
except ValueError:
raise optparse.OptionValueError("%s expects parameter=value"
% opt_str)
else:
qp[option.dest] = value
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION,
add_help_option=False)
parser.set_defaults(
url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5,
max_lines=100,
jwt_file=DEFAULT_JWT_LOCATION)
parser.add_option("-h", "--help", action="store_true", default=False,
help="show help message and exit")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="verbose mode")
parser.add_option("-u", "--url", type="string",
help="URL of routing service (default %default)")
parser.add_option("-y", "--service", type="string", action="callback",
callback=add_qp,
help="target service (default dataselect)")
parser.add_option("-N", "--network", type="string", action="callback",
callback=add_qp,
help="network code or pattern")
parser.add_option("-S", "--station", type="string", action="callback",
callback=add_qp,
help="station code or pattern")
parser.add_option("-L", "--location", type="string", action="callback",
callback=add_qp,
help="location code or pattern")
parser.add_option("-C", "--channel", type="string", action="callback",
callback=add_qp,
help="channel code or pattern")
parser.add_option("-s", "--starttime", type="string", action="callback",
callback=add_qp,
help="start time")
parser.add_option("-e", "--endtime", type="string", action="callback",
callback=add_qp,
help="end time")
parser.add_option("-q", "--query", type="string", action="callback",
callback=add_qp, metavar="PARAMETER=VALUE",
help="additional query parameter")
parser.add_option("-t", "--timeout", type="int",
help="request timeout in seconds (default %default)")
parser.add_option("-r", "--retries", type="int",
help="number of retries (default %default)")
parser.add_option("-w", "--retry-wait", type="int",
help="seconds to wait before each retry "
"(default %default)")
parser.add_option("-n", "--threads", type="int",
help="maximum number of download threads "
"(default %default)")
parser.add_option("-c", "--credentials-file", type="string",
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a", "--auth-file", type="string",
help="file that contains the EIDA legacy auth token")
if _jwt_supported:
parser.add_option("-j", "--jwt-file", type="string",
help="file that contains the EIDA JWT token")
parser.add_option("-p", "--post-file", type="string",
help="request file in FDSNWS POST format")
parser.add_option("-f", "--arclink-file", type="string",
help="request file in ArcLink format")
parser.add_option("-b", "--breqfast-file", type="string",
help="request file in breq_fast format")
parser.add_option("-o", "--output-file", type="string",
help="file where downloaded data is written")
parser.add_option("-l", "--max-lines", type="int",
help="max lines per request (default %default)")
parser.add_option("-z", "--no-citation", action="store_true", default=False,
help="suppress network citation info")
parser.add_option("-Z", "--no-check", action="store_true", default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if options.help:
parser.print_help()
return 0
if args or not options.output_file:
parser.print_usage(sys.stderr)
return 1
if bool(options.post_file) + bool(options.arclink_file) + \
bool(options.breqfast_file) > 1:
msg("only one of (--post-file, --arclink-file, --breqfast-file) "
"can be used")
return 1
try:
cred = {}
authdata = None
postdata = None
chans_to_check = set()
if options.credentials_file:
with open(options.credentials_file) as fd:
try:
for (url, user, passwd) in csv.reader(fd):
cred[url] = (user, passwd)
except (ValueError, csv.Error):
raise Error("error parsing %s" % options.credentials_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.credentials_file)
if options.auth_file:
with open(options.auth_file, 'rb') as fd:
authdata = fd.read()
else:
try:
with open(DEFAULT_TOKEN_LOCATION, 'rb') as fd:
authdata = fd.read()
options.auth_file = DEFAULT_TOKEN_LOCATION
except IOError:
pass
if authdata:
msg("using EIDA legacy auth token in %s" % options.auth_file, options.verbose)
try:
proc = subprocess.Popen(['gpg', '--decrypt'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(authdata)
if not out:
if isinstance(err, bytes):
err = err.decode('utf-8')
msg(err)
return 1
if isinstance(out, bytes):
out = out.decode('utf-8')
msg(out, options.verbose)
except OSError as e:
msg(str(e))
elif _jwt_supported and options.jwt_file:
if os.path.exists(options.jwt_file):
msg("using EIDA JWT token in %s" % options.jwt_file, options.verbose)
else:
msg("%s does not exist -- JWT auth disabled" % options.jwt_file)
options.jwt_file = None
if options.post_file:
try:
with open(options.post_file) as fd:
postdata = fd.read()
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.post_file)
else:
parser = None
if options.arclink_file:
parser = ArclinkParser()
try:
parser.parse(options.arclink_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.arclink_file)
elif options.breqfast_file:
parser = BreqParser()
try:
parser.parse(options.breqfast_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s"
% options.breqfast_file)
if parser is not None:
if parser.failstr:
msg(parser.failstr)
return 1
postdata = parser.postdata
if not options.no_check:
if postdata:
for line in postdata.splitlines():
nslc = line.split()[:4]
if nslc[0].startswith('_'): # virtual net
continue
if nslc[2] == '--': nslc[2] = ''
chans_to_check.add('.'.join(nslc))
else:
net = qp.get('network', '*')
sta = qp.get('station', '*')
loc = qp.get('location', '*')
cha = qp.get('channel', '*')
for n in net.split(','):
if n.startswith('_'): # virtual net
continue
for s in sta.split(','):
for l in loc.split(','):
for c in cha.split(','):
if l == '--': l = ''
chans_to_check.add('.'.join((n, s, l, c)))
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = open(options.output_file, 'wb')
nets = route(url, cred, authdata, options.jwt_file, postdata, dest,
chans_to_check, options.timeout, options.retries,
options.retry_wait, options.threads, options.max_lines,
options.verbose)
if nets and not options.no_citation:
msg("retrieving network citation info", options.verbose)
get_citation(nets, options)
else:
msg("", options.verbose)
msg("In case of problems with your request, plese use the contact "
"form at\n\n"
" http://www.orfeus-eu.org/organization/contact/form/"
"?recipient=EIDA\n", options.verbose)
except (IOError, Error) as e:
msg(str(e))
return 1
return 0
if __name__ == "__main__":
__doc__ %= {"prog": sys.argv[0]}
sys.exit(main())