Commit 5c17c921 authored by Franziska Koehn's avatar Franziska Koehn
Browse files

Merge branch 'feature/webinterface' into develop

parents c2d24776 e021816c
......@@ -5,3 +5,4 @@
"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}"},
{"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"},
{"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"},
{"label":"Type", "field":"xnat:mrScanData/TYPE", "required":true, "key": "type"},
{"label":"Project-ID", "field":"xnat:mrSessionData/PROJECT", "required":true, "key":"xnat_mrsessiondata_project"},
{"label":"Session-ID", "field":"xnat:mrSessionData/SESSION_ID", "required":false, "key": "session_id"},
{"label":"Date", "field":"xnat:mrSessionData/DATE", "required":false, "key": "date"},
{"label":"Project-ID", "field":"xnat:projectData/ID", "required":true, "key": "id"},
{"label":"NAME", "field":"xnat:projectData/NAME", "required":false, "key": "name"},
{"label":"Subject-ID", "field":"xnat:subjectData/SUBJECT_ID", "required":true, "key": "subject_id"},
{"label":"Subject-Label", "field":"xnat:subjectData/SUBJECT_LABEL", "required":false, "key": "subject_label"},
......@@ -39,25 +39,13 @@ def get_all(force=False):
return get_all()
get_all.cache = None
def get_comparison_extra_source(root_type):
def get_extra_source(root_type):
for data in get_all():
if data['root-type'] == root_type:
if not 'comparison_data' in data or not data['comparison_data']:
raise NoComparisonDataError("No comparison-data was defined for this datatype")
elif not 'extra-source' in data['comparison_data'] or not data['comparison_data']['extra-source']:
raise NoComparisonDataError("No Source of additional Data was defined for this datatype")
if not 'extra-source' in data:
raise NoExtraSourceError("No Source of additional Data was defined for this datatype")
return data['comparison_data']['extra-source']
def get_comparison_identifier(root_type):
for data in get_all():
if data['root-type'] == root_type:
if not 'comparison_data' in data or not data['comparison_data']:
raise NoComparisonDataError("No comparison-data was defined for this datatype")
elif not 'identifier' in data['comparison_data'] or not data['comparison_data']['identifier']:
raise NoComparisonDataError("No Identifier for this datatype was defined")
return data['comparison_data']['identifier']
return data['extra-source']
def get_rest(root_type):
......@@ -81,7 +69,7 @@ def get_root_types():
def get_fields(root_type):
"""Returns all fields of a given Root-Type.
"""Returns all fields (including key, label, field,...) of a given Root-Type.
:root_type: str, Root-Type of returned fields
......@@ -91,6 +79,18 @@ def get_fields(root_type):
return data['fields']
def get_field_list(root_type):
"""Returns a list of fields of a given root_type.
:root_type: str, Root-Type of returned field-labels
fields = get_fields(root_type)
if fields is not None:
return list(f['field'] for f in fields)
return []
def get_labels(root_type):
"""Returns a list of field-labels of a given root_type.
......@@ -129,6 +129,34 @@ def get_field_label_by_key(root_type, field_key):
return field_key
def get_field_by_key(root_type, field_key):
"""Returns the field of the given field-key.
:root_type: str, Root-Type of given key
:field_key: str, field-Key, for which the label will be returned
fields = get_fields(root_type)
for f in fields:
if "key" in f and f["key"] == field_key:
return f["field"]
return field_key
def get_field_key_by_label(root_type, label):
"""Returns the key of the given field-label.
:root_type: str, Root-Type of label
:label: str, field-label, for which the key will be returned
fields = get_fields(root_type)
for f in fields:
if f['label'] == label:
return f["key"]
return label
def get_field_label_by_field(root_type, field):
Returns the label of the given field.
......@@ -143,6 +171,20 @@ def get_field_label_by_field(root_type, field):
return f['label']
return field
def get_field_by_label(root_type, label):
Returns the field of the given label.
:root_type: str, Root-Type of given field
:label: str, label for which the field will be returned
fields = get_fields(root_type)
for f in fields:
if f['label'] == label:
return f['field']
return label
def get_fields_required(root_type):
"""Returns a list, containing a tuple in which the first value is the label of a field and the second the boolean, if its required or not.
......@@ -38,7 +38,7 @@ class NoRestApiError(Error):
"""Should be raised when no REST-API was defined in the json-file of a root-type"""
class NoComparisonDataError(Error):
class NoExtraSourceError(Error):
"""Should be raised when no comparision was defined in the json-file of a root-type"""
......@@ -41,7 +41,7 @@ def get_column(jdata, col, val_pattern='*'):
jdata = [jdata]
if val_pattern == '*':
return [entry[col] for entry in jdata if entry.has_key(col)]
return [entry[col] for entry in jdata if col in entry]
return [entry[col] for entry in jdata
if fnmatch(entry.get(col), val_pattern)
......@@ -159,7 +159,7 @@ class JsonTable(object):
return iter(
def __getitem__(self, name):
if isinstance(name, (str, unicode)):
if isinstance(name, str):
return self.get(name)
elif isinstance(name, int):
return self.__class__([[name]], self.order_by)
......@@ -94,7 +94,7 @@ def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb
ret = re.sub('\{\w+\}', subfunc, rest)
path = os.path.join(dest_folder, '-'.join(names))
url = "%s%s" % (host, ret)
url = "%s%s" % (host.strip("/"), ret)
download_file(url, creds, path)
......@@ -138,6 +138,45 @@ def download_file(url, creds, path):
raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
return False
def download_file_iter(url, creds):
Downloads a file from given 'url'.
Returns Iter of response
Raises xsa.errors.WritingError and xsa.errors.DownloadError
:url: str, host/REST-API with filled values
:creds: tuple(user-name, user-password), credentials
import requests
from base64 import b64encode
from requests.auth import HTTPBasicAuth
user, passw = creds
response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
if not response.ok:
raise ValueError(response.status_code)
return response.headers.get("content-length", None), response.iter_content(1024)
except IOError as e:
raise xsa.errors.WritingError("Error writing file %s" % e)
except ValueError as e:
raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
def prepare_rest(result, rest, host):
import re
def subfunc(f):
key ="{}")
return result[key]
ret = re.sub('\{\w+\}', subfunc, rest)
filename = ret.rpartition("/")[0].strip("/").replace("/","_")
return "%s%s" % (host.strip('/'), ret), filename
def retry_qry(fun):
def retry(*args, **kwargs):
last_exception = None
......@@ -146,7 +185,6 @@ def retry_qry(fun):
return fun(*args, **kwargs)
except Exception as e:
print(("wrapped", e))
if type(last_exception) is type(e):
raise e
last_exception = e
......@@ -167,7 +205,7 @@ def query_for_additional_data(rest, result, host, creds):
return result[key]
ret = re.sub('\{\w+\}', subfunc, rest)
user, passw = creds
url = "%s%s" % (host, ret)
url = "%s%s" % (host.strip("/"), ret)
r = requests.get(url, auth=HTTPBasicAuth(user, passw))
exp = ElementTree.fromstring(r.text)
......@@ -180,7 +218,6 @@ def query_for_additional_data(rest, result, host, creds):
new_result[fieldname] = ''.join(value.text).replace("\n","")
return new_result
def search_for(host, root_element, constraints, search_fields, user, passw):
......@@ -207,18 +244,33 @@ def search_for(host, root_element, constraints, search_fields, user, passw):
from xsa.jsonutil import JsonTable
import json
url= "%s%s" % (host, "/data/search?format=json")
url = "%s%s" % (host.strip("/"), "/data/search?format=json")
with NamedTemporaryFile(suffix=".xml") as tmp:
tree = create_xml_tree(root_element, constraints, search_fields)
tree.write(tmp, xml_declaration=True)
r =, files={'file': open(, 'rb')}, auth=HTTPBasicAuth(user, passw))
r =, files={'file': open(, 'rb')}, auth=HTTPBasicAuth(user, passw))
except requests.exceptions.InvalidSchema as e:
raise xsa.errors.ServerNotFoundError(e)
except requests.exceptions.InvalidURL as e:
raise xsa.errors.ServerNotFoundError(e)
except requests.RequestException as e:
raise xsa.errors.Error(e)
if r.status_code ==
return JsonTable(json_file)
elif r.status_code ==
raise xsa.errors.UnauthorizedError("HTTP 401: unauthorized")
elif r.status_code ==
raise xsa.errors.Error("HTTP 403: forbidden (check query definition, e.g. roottype)")
elif r.status_code ==
raise xsa.errors.ServerNotFoundError("HTTP 404: not found (check host-address)")
elif r.status_code ==
raise xsa.errors.Error("HTTP 500: Internal Server Error (check query definition, e.g. constraints)")
def create_xml_tree(root_type, query, fields):
import xml.etree.cElementTree as ET
......@@ -7,5 +7,7 @@ if __name__ == "__main__":
parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="http://localhost:8080")
parser.add_argument('--user', type=str, help='user:passw', default=":")
from xsagtk.xsa_app_main import start_xsa_gui
args = parser.parse_args()
from xsaweb.main import start_web_gui
#!/usr/bin/env python
import xsa.queries as queries
def main(host, user, passw):
import distutils.core
#TODO Read query from json-file
root_element = 'xnat:mrScanData'
constraints = [
('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
[('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
search_fields=[ 'xnat:mrScanData/TYPE',
results = queries.search_for(host, root_element, constraints,search_fields, user, passw)
print "Search results (%s):" % len(results)
print results
while True:
is_downl = distutils.util.strtobool(raw_input("download? "))
except ValueError:
print "invalid character. Please use y or n."
if is_downl:
print "downloading files..."
for result in results:
print result
queries.download_async(result, host, (user, passw), rest, dest_folder='')
print "finished"
print "download canceled"
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='search and download')
parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
parser.add_argument('--user', type=str, help='user:passw', default=":")
args = parser.parse_args()
u,p = args.user.split(':')
main(, u, p)
#!/usr/bin/env python
from __future__ import print_function
import argparse
import getpass
import sys
import requests
import json
import xsa.errors
import xsa.queries as queries
import xsa.datatypereader as datatypereader
from requests.auth import HTTPBasicAuth
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='search and download')
parser.add_argument('host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
parser.add_argument('user', type=str, help='username')
parser.add_argument('query', type=argparse.FileType('r'), default=sys.stdin, nargs="?", help='query-file includes root-type, query, and requested fieldnames (stdin if left out)')
parser.add_argument('--csv_out', '-o', metavar="<filename>", type=str, default=None,
help='name of the result-csv-file. If not set, no result-file will be generated and the regex will be ignored.')
parser.add_argument('--passw', '-p', type=str, default=None, help='Password of your user')
args = parser.parse_args()
host ='/')
user = args.user
query = json.load(args.query)
csv_out = args.csv_out
passw = args.passw
if not passw:
while True:
passw = getpass.getpass('Password for %s:' % user)
if requests.get(host, auth=HTTPBasicAuth(user, passw)).status_code != 200:
print("credentials are not correct")
print("password accepted")
root_element = query['root']
def convert(l):
if isinstance(l, str):
return l
elif len(l) == 3 and all(isinstance(e, str) for e in l):
return tuple(l)
return list(convert(e) for e in l)
constraints = convert(query['query'])
fields = query.get('fields', None) or query.get('labels')
search_fields = []
if fields:
for field in fields:
if field in datatypereader.get_labels(root_element):
search_fields.append(datatypereader.get_field_by_label(root_element, field))
elif field in datatypereader.get_keys(root_element):
search_fields.append(datatypereader.get_field_by_key(root_element, field))
elif field in datatypereader.get_field_list(root_element):
print('cant use this: %s , removed it from list of requested fields!', field)
raise xsa.errors.QueryError('No search-fields are defined in the passed query (use <fields> or <labels> as key)!')
results = queries.search_for(host, root_element, constraints, search_fields, user, passw)
if csv_out:
results.dump_csv(csv_out, delimiter="|")
print("saved to %s" % csv_out)
:Author: Franziska Koehn
:Created: 2015/01/13
This module houses classes relating to the GUI-representation of the chart.
from gi.repository import Gtk
from gi.repository import GObject
class ChartView(Gtk.HPaned):
"""Container with all widgets for working with the chart."""
results = []
bar_chart = None
canvas = None
def __init__(self):
"""Creates chart-area."""
super(ChartView, self).__init__()
v_box = Gtk.VBox()
hBox_type = Gtk.HBox()
v_box.pack_start(hBox_type, False, True, 0)
label_type = Gtk.Label()
label_type.set_text("Type: ")
hBox_type.pack_start(label_type, False, True, 0)
self.combobox = Gtk.ComboBox.new_with_model_and_entry(Gtk.ListStore(str, str))
hBox_type.pack_start(self.combobox, True, True, 0)
sw_hist_values = Gtk.ScrolledWindow()
v_box.pack_start(sw_hist_values, True, True, 0)
self.TreeViewChartValues = TreeViewChartValues()
changed_cb = lambda *_: self.update_chart()
self.combobox.connect('changed', changed_cb)
self.TreeViewChartValues.connect("values-changed", changed_cb)
from matplotlib.backends.backend_gtk3cairo import FigureCanvasGTK3Cairo as FigureCanvas
self.canvas = FigureCanvas(self.bar_chart)
def update_chart(self):
"""Creates the chart from all values from TreeViewChartValues"""
import xsa.chart as chart
substrings = []
strings = []
it = self.combobox.get_active_iter()
if it is not None:
field = self.combobox.get_model()[it][1]
for r in self.results:
for sub in
if not sub[0] == self.TreeViewChartValues.inital_value:
data = chart.count_substrings(substrings, strings)
self.bar_chart = chart.create(data,len(self.results), len(, self.bar_chart)
if self.canvas is not None:
def update(self, data, root_type):
Updates chart and combobox of types for given Root-Type and results.
:data: JsonTable, results of search
:root_type: str, Root-Type of search
self.results = data
import xsa.datatypereader as type_reader
store = Gtk.ListStore(str, str)
if data != []:
for key in data.headers():
label = type_reader.get_field_label_by_key(root_type, key)
store.append([label, key])
class TreeViewChartValues(Gtk.TreeView, GObject.GObject):
"""TreeView for creating the Search-Strings, shown in the chart."""
inital_value = "..."
"""Start-string for new inserted row."""
tooltip = 'Delete by using right click'
"""Tooltip that will be shown for each row and each column"""
__gsignals__= {'values-changed': (GObject.SIGNAL_RUN_FIRST, None,())}
def __init__(self):
"""Creats the model and column."""
super(TreeViewChartValues, self).__init__()
def on_treeview_button_press_event(treeview, event):
if event.button == 3:
x = int(event.x)
y = int(event.y)
path_info = treeview.get_path_at_pos(x, y)
if path_info is not None:
path, col, cellx, celly = path_info
iter =
if,0) != self.inital_value:
self.connect('button_press_event', on_treeview_button_press_event)
def create_column(self):
"""Creates column and its edited-callback."""
def cell_edited_callback(cellrenderertext, path_string, new_text, *_):
it =
it_last =, 0, new_text)