Commit 5c17c921 authored by Franziska Koehn's avatar Franziska Koehn
Browse files

Merge branch 'feature/webinterface' into develop

parents c2d24776 e021816c
...@@ -5,3 +5,4 @@ ...@@ -5,3 +5,4 @@
*.log *.log
*~ *~
doc/build doc/build
venv/
{ {
"root-type":"ext:cSubAssessment", "root-type":"ext:cSubAssessment",
"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}"}, "extra-source":"/data/archive/projects/{project}/experiments/{expt_id}",
"fields":[ "fields":[
{"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"}, {"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"},
{"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"}, {"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"},
......
{ {
"root-type":"xnat:mrScanData", "root-type":"xnat:mrScanData",
"comparison_data":{"identifier":["id","xnat_mrsessiondata_session_id"]}, "REST-API":"/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/<DICOM,NIFTI>/files?format=zip",
"REST-API":"/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/DICOM/files?format=zip",
"fields":[ "fields":[
{"label":"Type", "field":"xnat:mrScanData/TYPE", "required":true, "key": "type"}, {"label":"Type", "field":"xnat:mrScanData/TYPE", "required":true, "key": "type"},
{"label":"Project-ID", "field":"xnat:mrSessionData/PROJECT", "required":true, "key":"xnat_mrsessiondata_project"}, {"label":"Project-ID", "field":"xnat:mrSessionData/PROJECT", "required":true, "key":"xnat_mrsessiondata_project"},
......
{ {
"root-type":"xnat:mrSessionData", "root-type":"xnat:mrSessionData",
"comparison_data":{"identifier":["session_id"]},
"fields":[ "fields":[
{"label":"Session-ID", "field":"xnat:mrSessionData/SESSION_ID", "required":false, "key": "session_id"}, {"label":"Session-ID", "field":"xnat:mrSessionData/SESSION_ID", "required":false, "key": "session_id"},
{"label":"Date", "field":"xnat:mrSessionData/DATE", "required":false, "key": "date"}, {"label":"Date", "field":"xnat:mrSessionData/DATE", "required":false, "key": "date"},
......
{ {
"root-type":"xnat:projectData", "root-type":"xnat:projectData",
"comparison_data":{"identifier":["id"]},
"fields":[ "fields":[
{"label":"Project-ID", "field":"xnat:projectData/ID", "required":true, "key": "id"}, {"label":"Project-ID", "field":"xnat:projectData/ID", "required":true, "key": "id"},
{"label":"NAME", "field":"xnat:projectData/NAME", "required":false, "key": "name"}, {"label":"NAME", "field":"xnat:projectData/NAME", "required":false, "key": "name"},
......
{ {
"root-type":"xnat:subjectData", "root-type":"xnat:subjectData",
"comparison_data":{"identifier":["subject_id"]},
"fields":[ "fields":[
{"label":"Subject-ID", "field":"xnat:subjectData/SUBJECT_ID", "required":true, "key": "subject_id"}, {"label":"Subject-ID", "field":"xnat:subjectData/SUBJECT_ID", "required":true, "key": "subject_id"},
{"label":"Subject-Label", "field":"xnat:subjectData/SUBJECT_LABEL", "required":false, "key": "subject_label"}, {"label":"Subject-Label", "field":"xnat:subjectData/SUBJECT_LABEL", "required":false, "key": "subject_label"},
......
httplib2 Flask==0.10.1
lxml requests==2.7.0
pyxnat
requests
pydash
matplotlib
...@@ -39,25 +39,13 @@ def get_all(force=False): ...@@ -39,25 +39,13 @@ def get_all(force=False):
return get_all() return get_all()
get_all.cache = None get_all.cache = None
def get_comparison_extra_source(root_type): def get_extra_source(root_type):
for data in get_all(): for data in get_all():
if data['root-type'] == root_type: if data['root-type'] == root_type:
if not 'comparison_data' in data or not data['comparison_data']: if not 'extra-source' in data:
raise NoComparisonDataError("No comparison-data was defined for this datatype") raise NoExtraSourceError("No Source of additional Data was defined for this datatype")
elif not 'extra-source' in data['comparison_data'] or not data['comparison_data']['extra-source']:
raise NoComparisonDataError("No Source of additional Data was defined for this datatype")
else: else:
return data['comparison_data']['extra-source'] return data['extra-source']
def get_comparison_identifier(root_type):
for data in get_all():
if data['root-type'] == root_type:
if not 'comparison_data' in data or not data['comparison_data']:
raise NoComparisonDataError("No comparison-data was defined for this datatype")
elif not 'identifier' in data['comparison_data'] or not data['comparison_data']['identifier']:
raise NoComparisonDataError("No Identifier for this datatype was defined")
else:
return data['comparison_data']['identifier']
def get_rest(root_type): def get_rest(root_type):
...@@ -81,7 +69,7 @@ def get_root_types(): ...@@ -81,7 +69,7 @@ def get_root_types():
def get_fields(root_type): def get_fields(root_type):
"""Returns all fields of a given Root-Type. """Returns all fields (including key, label, field,...) of a given Root-Type.
**Parameters** **Parameters**
:root_type: str, Root-Type of returned fields :root_type: str, Root-Type of returned fields
...@@ -91,6 +79,18 @@ def get_fields(root_type): ...@@ -91,6 +79,18 @@ def get_fields(root_type):
return data['fields'] return data['fields']
def get_field_list(root_type):
"""Returns a list of fields of a given root_type.
**Parameters**
:root_type: str, Root-Type of returned field-labels
"""
fields = get_fields(root_type)
if fields is not None:
return list(f['field'] for f in fields)
return []
def get_labels(root_type): def get_labels(root_type):
"""Returns a list of field-labels of a given root_type. """Returns a list of field-labels of a given root_type.
...@@ -129,6 +129,34 @@ def get_field_label_by_key(root_type, field_key): ...@@ -129,6 +129,34 @@ def get_field_label_by_key(root_type, field_key):
return field_key return field_key
def get_field_by_key(root_type, field_key):
"""Returns the field of the given field-key.
**Parameters**
:root_type: str, Root-Type of given key
:field_key: str, field-Key, for which the label will be returned
"""
fields = get_fields(root_type)
for f in fields:
if "key" in f and f["key"] == field_key:
return f["field"]
return field_key
def get_field_key_by_label(root_type, label):
"""Returns the key of the given field-label.
**Parameters**
:root_type: str, Root-Type of label
:label: str, field-label, for which the key will be returned
"""
fields = get_fields(root_type)
for f in fields:
if f['label'] == label:
return f["key"]
return label
def get_field_label_by_field(root_type, field): def get_field_label_by_field(root_type, field):
""" """
Returns the label of the given field. Returns the label of the given field.
...@@ -143,6 +171,20 @@ def get_field_label_by_field(root_type, field): ...@@ -143,6 +171,20 @@ def get_field_label_by_field(root_type, field):
return f['label'] return f['label']
return field return field
def get_field_by_label(root_type, label):
"""
Returns the field of the given label.
**Parameters**
:root_type: str, Root-Type of given field
:label: str, label for which the field will be returned
"""
fields = get_fields(root_type)
for f in fields:
if f['label'] == label:
return f['field']
return label
def get_fields_required(root_type): def get_fields_required(root_type):
"""Returns a list, containing a tuple in which the first value is the label of a field and the second the boolean, if its required or not. """Returns a list, containing a tuple in which the first value is the label of a field and the second the boolean, if its required or not.
......
...@@ -38,7 +38,7 @@ class NoRestApiError(Error): ...@@ -38,7 +38,7 @@ class NoRestApiError(Error):
"""Should be raised when no REST-API was defined in the json-file of a root-type""" """Should be raised when no REST-API was defined in the json-file of a root-type"""
pass pass
class NoComparisonDataError(Error): class NoExtraSourceError(Error):
"""Should be raised when no comparision was defined in the json-file of a root-type""" """Should be raised when no comparision was defined in the json-file of a root-type"""
pass pass
......
...@@ -41,7 +41,7 @@ def get_column(jdata, col, val_pattern='*'): ...@@ -41,7 +41,7 @@ def get_column(jdata, col, val_pattern='*'):
jdata = [jdata] jdata = [jdata]
if val_pattern == '*': if val_pattern == '*':
return [entry[col] for entry in jdata if entry.has_key(col)] return [entry[col] for entry in jdata if col in entry]
else: else:
return [entry[col] for entry in jdata return [entry[col] for entry in jdata
if fnmatch(entry.get(col), val_pattern) if fnmatch(entry.get(col), val_pattern)
...@@ -159,7 +159,7 @@ class JsonTable(object): ...@@ -159,7 +159,7 @@ class JsonTable(object):
return iter(self.data) return iter(self.data)
def __getitem__(self, name): def __getitem__(self, name):
if isinstance(name, (str, unicode)): if isinstance(name, str):
return self.get(name) return self.get(name)
elif isinstance(name, int): elif isinstance(name, int):
return self.__class__([self.data[name]], self.order_by) return self.__class__([self.data[name]], self.order_by)
......
...@@ -94,7 +94,7 @@ def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb ...@@ -94,7 +94,7 @@ def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb
ret = re.sub('\{\w+\}', subfunc, rest) ret = re.sub('\{\w+\}', subfunc, rest)
path = os.path.join(dest_folder, '-'.join(names)) path = os.path.join(dest_folder, '-'.join(names))
url = "%s%s" % (host, ret) url = "%s%s" % (host.strip("/"), ret)
download_file(url, creds, path) download_file(url, creds, path)
requests_lock.release() requests_lock.release()
...@@ -138,6 +138,45 @@ def download_file(url, creds, path): ...@@ -138,6 +138,45 @@ def download_file(url, creds, path):
raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e)) raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
return False return False
def download_file_iter(url, creds):
"""
Downloads a file from given 'url'.
Returns Iter of response
Raises xsa.errors.WritingError and xsa.errors.DownloadError
**Parameters**
:url: str, host/REST-API with filled values
:creds: tuple(user-name, user-password), credentials
"""
import requests
from base64 import b64encode
from requests.auth import HTTPBasicAuth
user, passw = creds
try:
response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
if not response.ok:
raise ValueError(response.status_code)
return response.headers.get("content-length", None), response.iter_content(1024)
except IOError as e:
raise xsa.errors.WritingError("Error writing file %s" % e)
except ValueError as e:
raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
def prepare_rest(result, rest, host):
import re
def subfunc(f):
key = f.group(0).strip("{}")
return result[key]
ret = re.sub('\{\w+\}', subfunc, rest)
filename = ret.rpartition("/")[0].strip("/").replace("/","_")
return "%s%s" % (host.strip('/'), ret), filename
def retry_qry(fun): def retry_qry(fun):
def retry(*args, **kwargs): def retry(*args, **kwargs):
last_exception = None last_exception = None
...@@ -146,7 +185,6 @@ def retry_qry(fun): ...@@ -146,7 +185,6 @@ def retry_qry(fun):
try: try:
return fun(*args, **kwargs) return fun(*args, **kwargs)
except Exception as e: except Exception as e:
print(("wrapped", e))
if type(last_exception) is type(e): if type(last_exception) is type(e):
raise e raise e
last_exception = e last_exception = e
...@@ -167,7 +205,7 @@ def query_for_additional_data(rest, result, host, creds): ...@@ -167,7 +205,7 @@ def query_for_additional_data(rest, result, host, creds):
return result[key] return result[key]
ret = re.sub('\{\w+\}', subfunc, rest) ret = re.sub('\{\w+\}', subfunc, rest)
user, passw = creds user, passw = creds
url = "%s%s" % (host, ret) url = "%s%s" % (host.strip("/"), ret)
r = requests.get(url, auth=HTTPBasicAuth(user, passw)) r = requests.get(url, auth=HTTPBasicAuth(user, passw))
exp = ElementTree.fromstring(r.text) exp = ElementTree.fromstring(r.text)
...@@ -180,7 +218,6 @@ def query_for_additional_data(rest, result, host, creds): ...@@ -180,7 +218,6 @@ def query_for_additional_data(rest, result, host, creds):
new_result[fieldname] = ''.join(value.text).replace("\n","") new_result[fieldname] = ''.join(value.text).replace("\n","")
return new_result return new_result
@retry_qry @retry_qry
def search_for(host, root_element, constraints, search_fields, user, passw): def search_for(host, root_element, constraints, search_fields, user, passw):
""" """
...@@ -207,18 +244,33 @@ def search_for(host, root_element, constraints, search_fields, user, passw): ...@@ -207,18 +244,33 @@ def search_for(host, root_element, constraints, search_fields, user, passw):
from xsa.jsonutil import JsonTable from xsa.jsonutil import JsonTable
import json import json
url= "%s%s" % (host, "/data/search?format=json") url = "%s%s" % (host.strip("/"), "/data/search?format=json")
with NamedTemporaryFile(suffix=".xml") as tmp: with NamedTemporaryFile(suffix=".xml") as tmp:
tree = create_xml_tree(root_element, constraints, search_fields) tree = create_xml_tree(root_element, constraints, search_fields)
tree.write(tmp, xml_declaration=True) tree.write(tmp, xml_declaration=True)
tmp.flush() tmp.flush()
r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw)) try:
r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))
except requests.exceptions.InvalidSchema as e:
raise xsa.errors.ServerNotFoundError(e)
except requests.exceptions.InvalidURL as e:
raise xsa.errors.ServerNotFoundError(e)
except requests.RequestException as e:
raise xsa.errors.Error(e)
print("response=",r.status_code)
if r.status_code == requests.codes.ok: if r.status_code == requests.codes.ok:
json_file=(json.loads(r.text))['ResultSet']['Result'] json_file=(json.loads(r.text))['ResultSet']['Result']
return JsonTable(json_file) return JsonTable(json_file)
elif r.status_code == requests.codes.unauthorized:
raise xsa.errors.UnauthorizedError("HTTP 401: unauthorized")
elif r.status_code == requests.codes.forbidden:
raise xsa.errors.Error("HTTP 403: forbidden (check query definition, e.g. roottype)")
elif r.status_code == requests.codes.not_found:
raise xsa.errors.ServerNotFoundError("HTTP 404: not found (check host-address)")
elif r.status_code == requests.codes.internal_server_error:
raise xsa.errors.Error("HTTP 500: Internal Server Error (check query definition, e.g. constraints)")
def create_xml_tree(root_type, query, fields): def create_xml_tree(root_type, query, fields):
import xml.etree.cElementTree as ET import xml.etree.cElementTree as ET
......
...@@ -7,5 +7,7 @@ if __name__ == "__main__": ...@@ -7,5 +7,7 @@ if __name__ == "__main__":
parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="http://localhost:8080") parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="http://localhost:8080")
parser.add_argument('--user', type=str, help='user:passw', default=":") parser.add_argument('--user', type=str, help='user:passw', default=":")
from xsagtk.xsa_app_main import start_xsa_gui args = parser.parse_args()
start_xsa_gui(parser.parse_args())
from xsaweb.main import start_web_gui
start_web_gui(args)
#!/usr/bin/env python
import xsa.queries as queries
def main(host, user, passw):
import distutils.core
#TODO Read query from json-file
root_element = 'xnat:mrScanData'
constraints = [
('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
'AND',
[('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
'OR'
]
]
search_fields=[ 'xnat:mrScanData/TYPE',
'xnat:mrSessionData/PROJECT',
'xnat:mrSessionData/SUBJECT_ID',
'xnat:mrSessionData/SESSION_ID',
'xnat:mrScanData/ID'
]
rest="/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/DICOM/files?format=zip"
results = queries.search_for(host, root_element, constraints,search_fields, user, passw)
print "Search results (%s):" % len(results)
print results
while True:
try:
is_downl = distutils.util.strtobool(raw_input("download? "))
break
except ValueError:
print "invalid character. Please use y or n."
if is_downl:
print "downloading files..."
for result in results:
print result
queries.download_async(result, host, (user, passw), rest, dest_folder='')
print "finished"
else:
print "download canceled"
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='search and download')
parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
parser.add_argument('--user', type=str, help='user:passw', default=":")
args = parser.parse_args()
u,p = args.user.split(':')
main(args.host, u, p)
#!/usr/bin/env python
from __future__ import print_function
import argparse
import getpass
import sys
import requests
import json
import xsa.errors
import xsa.queries as queries
import xsa.datatypereader as datatypereader
from requests.auth import HTTPBasicAuth
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='search and download')
parser.add_argument('host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
parser.add_argument('user', type=str, help='username')
parser.add_argument('query', type=argparse.FileType('r'), default=sys.stdin, nargs="?", help='query-file includes root-type, query, and requested fieldnames (stdin if left out)')
parser.add_argument('--csv_out', '-o', metavar="<filename>", type=str, default=None,
help='name of the result-csv-file. If not set, no result-file will be generated and the regex will be ignored.')
parser.add_argument('--passw', '-p', type=str, default=None, help='Password of your user')
args = parser.parse_args()
host = args.host.strip('/')
user = args.user
query = json.load(args.query)
csv_out = args.csv_out
passw = args.passw
if not passw:
while True:
passw = getpass.getpass('Password for %s:' % user)
if requests.get(host, auth=HTTPBasicAuth(user, passw)).status_code != 200:
print("credentials are not correct")
else:
print("password accepted")
break
root_element = query['root']
def convert(l):
if isinstance(l, str):
return l
elif len(l) == 3 and all(isinstance(e, str) for e in l):
return tuple(l)
else:
return list(convert(e) for e in l)
constraints = convert(query['query'])
fields = query.get('fields', None) or query.get('labels')
search_fields = []
if fields:
for field in fields:
if field in datatypereader.get_labels(root_element):
search_fields.append(datatypereader.get_field_by_label(root_element, field))
elif field in datatypereader.get_keys(root_element):
search_fields.append(datatypereader.get_field_by_key(root_element, field))
elif field in datatypereader.get_field_list(root_element):
search_fields.append(field)
else:
print('cant use this: %s , removed it from list of requested fields!', field)
else:
raise xsa.errors.QueryError('No search-fields are defined in the passed query (use <fields> or <labels> as key)!')
results = queries.search_for(host, root_element, constraints, search_fields, user, passw)
print(results)
if csv_out:
results.dump_csv(csv_out, delimiter="|")
print("saved to %s" % csv_out)
"""
:Author: Franziska Koehn
:Created: 2015/01/13
This module houses classes relating to the GUI-representation of the chart.
"""
from gi.repository import Gtk
from gi.repository import GObject
class ChartView(Gtk.HPaned):
"""Container with all widgets for working with the chart."""
results = []
bar_chart = None
canvas = None
def __init__(self):
"""Creates chart-area."""
super(ChartView, self).__init__()
self.set_position(250)
v_box = Gtk.VBox()
self.add1(v_box)
hBox_type = Gtk.HBox()
v_box.pack_start(hBox_type, False, True, 0)
label_type = Gtk.Label()
label_type.set_text("Type: ")
hBox_type.pack_start(label_type, False, True, 0)
self.combobox = Gtk.ComboBox.new_with_model_and_entry(Gtk.ListStore(str, str))
self.combobox.set_entry_text_column(0)
hBox_type.pack_start(self.combobox, True, True, 0)
sw_hist_values = Gtk.ScrolledWindow()
v_box.pack_start(sw_hist_values, True, True, 0)
self.TreeViewChartValues = TreeViewChartValues()
sw_hist_values.add(self.TreeViewChartValues)
changed_cb