Merge branch 'feature/webinterface' into develop

.gitignore

@@ -5,3 +5,4 @@
 *.log
 *~
 doc/build
+venv/

datatypes/ext_cSubAssessment.JSON

 {
 "root-type":"ext:cSubAssessment",
-"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}"},
+"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}",
 "fields":[
 {"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"},
 {"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"},

datatypes/xnat_mrScanData.JSON

 {
 "root-type":"xnat:mrScanData",
-"comparison_data":{"identifier":["id","xnat_mrsessiondata_session_id"]},
-"REST-API":"/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/DICOM/files?format=zip",
+"REST-API":"/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/<DICOM,NIFTI>/files?format=zip",
 "fields":[
 {"label":"Type", "field":"xnat:mrScanData/TYPE", "required":true, "key": "type"},
 {"label":"Project-ID", "field":"xnat:mrSessionData/PROJECT", "required":true, "key":"xnat_mrsessiondata_project"},

datatypes/xnat_mrSessionData.JSON

 {
 "root-type":"xnat:mrSessionData",
-"comparison_data":{"identifier":["session_id"]},
 "fields":[
 {"label":"Session-ID", "field":"xnat:mrSessionData/SESSION_ID", "required":false, "key": "session_id"},
 {"label":"Date", "field":"xnat:mrSessionData/DATE", "required":false, "key": "date"},

datatypes/xnat_projectData.JSON

 {
 "root-type":"xnat:projectData",
-"comparison_data":{"identifier":["id"]},
 "fields":[
 {"label":"Project-ID", "field":"xnat:projectData/ID", "required":true, "key": "id"},
 {"label":"NAME", "field":"xnat:projectData/NAME", "required":false, "key": "name"},

datatypes/xnat_subjectData.JSON

 {
 "root-type":"xnat:subjectData",
-"comparison_data":{"identifier":["subject_id"]},
 "fields":[
 {"label":"Subject-ID", "field":"xnat:subjectData/SUBJECT_ID", "required":true, "key": "subject_id"},
 {"label":"Subject-Label", "field":"xnat:subjectData/SUBJECT_LABEL", "required":false, "key": "subject_label"},

prerequisites.txt

-httplib2
-lxml
-pyxnat
-requests
-pydash
-matplotlib
+Flask==0.10.1
+requests==2.7.0

xsa/datatypereader.py

@@ -39,25 +39,13 @@ def get_all(force=False):
     return get_all()
 get_all.cache = None
 
-def get_comparison_extra_source(root_type):
+def get_extra_source(root_type):
     for data in get_all():
         if data['root-type'] == root_type:
-            if not 'comparison_data' in data or not data['comparison_data']:
-                raise NoComparisonDataError("No comparison-data was defined for this datatype")
-            elif not 'extra-source' in data['comparison_data'] or not data['comparison_data']['extra-source']:
-                raise NoComparisonDataError("No Source of additional Data was defined for this datatype")
+            if  not 'extra-source' in data:
+                raise NoExtraSourceError("No Source of additional Data was defined for this datatype")
             else:
-                return data['comparison_data']['extra-source']
-
-def get_comparison_identifier(root_type):
-    for data in get_all():
-        if data['root-type'] == root_type:
-            if not 'comparison_data' in data or not data['comparison_data']:
-                raise NoComparisonDataError("No comparison-data was defined for this datatype")
-            elif not 'identifier' in data['comparison_data'] or not data['comparison_data']['identifier']:
-                raise NoComparisonDataError("No Identifier for this datatype was defined")
-            else:
-                return data['comparison_data']['identifier']
+                return data['extra-source']
 
 
 def get_rest(root_type):
@@ -81,7 +69,7 @@ def get_root_types():
 
 
 def get_fields(root_type):
-    """Returns all fields of a given Root-Type.
+    """Returns all fields (including key, label, field,...) of a given Root-Type.
 
     **Parameters**
         :root_type: str, Root-Type of returned fields
@@ -91,6 +79,18 @@ def get_fields(root_type):
             return data['fields']
 
 
+def get_field_list(root_type):
+    """Returns a list of fields of a given root_type.
+
+    **Parameters**
+        :root_type: str, Root-Type of returned field-labels
+    """
+    fields = get_fields(root_type)
+    if fields is not None:
+        return list(f['field'] for f in fields)
+    return []
+
+
 def get_labels(root_type):
     """Returns a list of field-labels of a given root_type.
 
@@ -129,6 +129,34 @@ def get_field_label_by_key(root_type, field_key):
     return field_key
 
 
+def get_field_by_key(root_type, field_key):
+    """Returns the field of the given field-key.
+
+    **Parameters**
+        :root_type: str, Root-Type of given key
+        :field_key: str, field-Key, for which the label will be returned
+    """
+    fields = get_fields(root_type)
+    for f in fields:
+        if "key" in f and f["key"] == field_key:
+            return f["field"]
+    return field_key
+
+
+def get_field_key_by_label(root_type, label):
+    """Returns the key of the given field-label.
+
+    **Parameters**
+        :root_type: str, Root-Type of label
+        :label: str, field-label, for which the key will be returned
+    """
+    fields = get_fields(root_type)
+    for f in fields:
+        if f['label'] == label:
+            return f["key"]
+    return label
+
+
 def get_field_label_by_field(root_type, field):
     """
     Returns the label of the given field.
@@ -143,6 +171,20 @@ def get_field_label_by_field(root_type, field):
             return f['label']
     return field
 
+def get_field_by_label(root_type, label):
+    """
+    Returns the field of the given label.
+
+    **Parameters**
+        :root_type: str, Root-Type of given field
+        :label: str, label for which the field will be returned
+    """
+    fields = get_fields(root_type)
+    for f in fields:
+        if f['label'] == label:
+            return f['field']
+    return label
+
 def get_fields_required(root_type):
     """Returns a list, containing a tuple in which the first value is the label of a field and the second the boolean, if its required or not.
 

xsa/errors.py

@@ -38,7 +38,7 @@ class NoRestApiError(Error):
     """Should be raised when no REST-API was defined in the json-file of a root-type"""
     pass
 
-class NoComparisonDataError(Error):
+class NoExtraSourceError(Error):
     """Should be raised when no comparision was defined in the json-file of a root-type"""
     pass
 

xsa/jsonutil.py

@@ -41,7 +41,7 @@ def get_column(jdata, col, val_pattern='*'):
         jdata = [jdata]
 
     if val_pattern == '*':
-        return [entry[col] for entry in jdata if entry.has_key(col)]
+        return [entry[col] for entry in jdata if col in entry]
     else:
         return [entry[col] for entry in jdata
                 if fnmatch(entry.get(col), val_pattern)
@@ -159,7 +159,7 @@ class JsonTable(object):
         return iter(self.data)
 
     def __getitem__(self, name):
-        if isinstance(name, (str, unicode)):
+        if isinstance(name, str):
             return self.get(name)
         elif isinstance(name, int):
             return self.__class__([self.data[name]], self.order_by)

xsa/queries.py

@@ -94,7 +94,7 @@ def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb
     ret = re.sub('\{\w+\}', subfunc, rest)
 
     path = os.path.join(dest_folder, '-'.join(names))
-    url = "%s%s" % (host, ret)
+    url = "%s%s" % (host.strip("/"), ret)
 
     download_file(url, creds, path)
     requests_lock.release()
@@ -138,6 +138,45 @@ def download_file(url, creds, path):
         raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
     return False
 
+def download_file_iter(url, creds):
+    """
+    Downloads a file from given 'url'.
+
+    Returns Iter of response
+    Raises xsa.errors.WritingError and xsa.errors.DownloadError
+
+    **Parameters**
+        :url: str, host/REST-API with filled values
+        :creds: tuple(user-name, user-password), credentials
+    """
+
+    import requests
+    from base64 import b64encode
+    from requests.auth import HTTPBasicAuth
+
+    user, passw = creds
+
+    try:
+        response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
+        if not response.ok:
+            raise ValueError(response.status_code)
+        return response.headers.get("content-length", None), response.iter_content(1024)
+    except IOError as e:
+        raise xsa.errors.WritingError("Error writing file %s" %  e)
+    except ValueError as e:
+        raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
+
+def prepare_rest(result, rest, host):
+    import re
+
+    def subfunc(f):
+        key = f.group(0).strip("{}")
+        return result[key]
+    ret = re.sub('\{\w+\}', subfunc, rest)
+    filename = ret.rpartition("/")[0].strip("/").replace("/","_")
+
+    return "%s%s" % (host.strip('/'), ret), filename
+
 def retry_qry(fun):
     def retry(*args, **kwargs):
         last_exception = None
@@ -146,7 +185,6 @@ def retry_qry(fun):
             try:
                 return fun(*args, **kwargs)
             except Exception as e:
-                print(("wrapped", e))
                 if type(last_exception) is type(e):
                     raise e
                 last_exception = e
@@ -167,7 +205,7 @@ def query_for_additional_data(rest, result, host, creds):
         return result[key]
     ret = re.sub('\{\w+\}', subfunc, rest)
     user, passw = creds
-    url = "%s%s" % (host, ret)
+    url = "%s%s" % (host.strip("/"), ret)
 
     r = requests.get(url, auth=HTTPBasicAuth(user, passw))
     exp = ElementTree.fromstring(r.text)
@@ -180,7 +218,6 @@ def query_for_additional_data(rest, result, host, creds):
         new_result[fieldname] = ''.join(value.text).replace("\n","")
     return new_result
 
-
 @retry_qry
 def search_for(host, root_element, constraints, search_fields, user, passw):
     """
@@ -207,18 +244,33 @@ def search_for(host, root_element, constraints, search_fields, user, passw):
     from xsa.jsonutil import JsonTable
     import json
 
-    url= "%s%s" % (host, "/data/search?format=json")
+    url = "%s%s" % (host.strip("/"), "/data/search?format=json")
 
     with NamedTemporaryFile(suffix=".xml") as tmp:
         tree = create_xml_tree(root_element, constraints, search_fields)
         tree.write(tmp, xml_declaration=True)
         tmp.flush()
+        try:
             r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))
-
+        except requests.exceptions.InvalidSchema as e:
+            raise xsa.errors.ServerNotFoundError(e)
+        except requests.exceptions.InvalidURL as e:
+            raise xsa.errors.ServerNotFoundError(e)
+        except requests.RequestException as e:
+            raise xsa.errors.Error(e)
+
+    print("response=",r.status_code)
     if r.status_code == requests.codes.ok:
         json_file=(json.loads(r.text))['ResultSet']['Result']
         return JsonTable(json_file)
-
+    elif r.status_code == requests.codes.unauthorized:
+        raise xsa.errors.UnauthorizedError("HTTP 401: unauthorized")
+    elif r.status_code == requests.codes.forbidden:
+        raise xsa.errors.Error("HTTP 403: forbidden (check query definition, e.g. roottype)")
+    elif r.status_code == requests.codes.not_found:
+        raise xsa.errors.ServerNotFoundError("HTTP 404: not found (check host-address)")
+    elif r.status_code == requests.codes.internal_server_error:
+        raise xsa.errors.Error("HTTP 500: Internal Server Error (check query definition, e.g. constraints)")
 
 def create_xml_tree(root_type, query, fields):
     import xml.etree.cElementTree as ET

xsa_gui.py

@@ -7,5 +7,7 @@ if __name__ == "__main__":
     parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="http://localhost:8080")
     parser.add_argument('--user', type=str, help='user:passw', default=":")
 
-    from xsagtk.xsa_app_main import start_xsa_gui
-    start_xsa_gui(parser.parse_args())
+    args = parser.parse_args()
+
+    from xsaweb.main import start_web_gui
+    start_web_gui(args)

xsa_script.py

-#!/usr/bin/env python
-import xsa.queries as queries
-
-def main(host, user, passw):
-    import distutils.core
-
-    #TODO Read query from json-file
-
-    root_element = 'xnat:mrScanData'
-    constraints = [
-        ('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
-        ('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
-        'AND',
-        [('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
-         ('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
-         'OR'
-         ]
-        ]
-
-    search_fields=[ 'xnat:mrScanData/TYPE',
-                    'xnat:mrSessionData/PROJECT',
-                    'xnat:mrSessionData/SUBJECT_ID',
-                    'xnat:mrSessionData/SESSION_ID',
-                    'xnat:mrScanData/ID'
-                    ]
-    rest="/data/archive/projects/{xnat_mrsessiondata_project}/subjects/{xnat_mrsessiondata_subject_id}/experiments/{xnat_mrsessiondata_session_id}/scans/{id}/resources/DICOM/files?format=zip"
-
-
-    results = queries.search_for(host, root_element, constraints,search_fields, user, passw)
-
-    print "Search results (%s):" % len(results)
-    print results
-
-    while True:
-        try:
-            is_downl = distutils.util.strtobool(raw_input("download? "))
-            break
-        except ValueError:
-            print "invalid character. Please use y or n."
-
-    if is_downl:
-        print "downloading files..."
-        for result in results:
-            print result
-            queries.download_async(result, host, (user, passw), rest, dest_folder='')
-        print "finished"
-    else:
-        print "download canceled"
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser(description='search and download')
-    parser.add_argument('--host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
-    parser.add_argument('--user', type=str, help='user:passw', default=":")
-
-    args = parser.parse_args()
-
-    u,p = args.user.split(':')
-    main(args.host, u, p)

xsa_script_query.py

+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import getpass
+import sys
+import requests
+import json
+import xsa.errors
+import xsa.queries as queries
+import xsa.datatypereader as datatypereader
+
+from requests.auth import HTTPBasicAuth
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description='search and download')
+    parser.add_argument('host', type=str, help='hostname or ip-address (including port).', default="localhost:8080")
+    parser.add_argument('user', type=str, help='username')
+    parser.add_argument('query', type=argparse.FileType('r'), default=sys.stdin, nargs="?", help='query-file includes root-type, query, and requested fieldnames (stdin if left out)')
+    parser.add_argument('--csv_out', '-o', metavar="<filename>", type=str, default=None,
+                        help='name of the result-csv-file. If not set, no result-file will be generated and the regex will be ignored.')
+    parser.add_argument('--passw', '-p', type=str, default=None, help='Password of your user')
+
+    args = parser.parse_args()
+    host = args.host.strip('/')
+    user = args.user
+    query = json.load(args.query)
+    csv_out = args.csv_out
+    passw = args.passw
+
+    if not passw:
+        while True:
+            passw = getpass.getpass('Password for %s:' % user)
+            if requests.get(host, auth=HTTPBasicAuth(user, passw)).status_code != 200:
+                print("credentials are not correct")
+            else:
+                print("password accepted")
+                break
+
+
+    root_element = query['root']
+
+    def convert(l):
+        if isinstance(l, str):
+            return l
+        elif len(l) == 3 and all(isinstance(e, str) for e in l):
+            return tuple(l)
+        else:
+            return list(convert(e) for e in l)
+
+    constraints = convert(query['query'])
+
+    fields = query.get('fields', None) or query.get('labels')
+    search_fields = []
+    if fields:
+        for field in fields:
+            if field in datatypereader.get_labels(root_element):
+                search_fields.append(datatypereader.get_field_by_label(root_element, field))
+            elif field in datatypereader.get_keys(root_element):
+                search_fields.append(datatypereader.get_field_by_key(root_element, field))
+            elif field in datatypereader.get_field_list(root_element):
+                search_fields.append(field)
+            else:
+                print('cant use this: %s , removed it from list of requested fields!', field)
+    else:
+        raise xsa.errors.QueryError('No search-fields are defined in the passed query (use <fields> or <labels> as key)!')
+
+    results = queries.search_for(host, root_element, constraints, search_fields, user, passw)
+
+    print(results)
+    if csv_out:
+        results.dump_csv(csv_out, delimiter="|")
+        print("saved to %s" % csv_out)

xsagtk/chart_view.py

-"""
-
-:Author: Franziska Koehn
-:Created: 2015/01/13
-
-This module houses classes relating to the GUI-representation of the chart.
-
-"""
-
-from gi.repository import Gtk
-from gi.repository import GObject
-
-class ChartView(Gtk.HPaned):
-    """Container with all widgets for working with the chart."""
-    results = []
-    bar_chart = None
-    canvas = None
-
-    def __init__(self):
-
-        """Creates chart-area."""
-
-        super(ChartView, self).__init__()
-
-        self.set_position(250)
-
-        v_box = Gtk.VBox()
-        self.add1(v_box)
-
-        hBox_type = Gtk.HBox()
-        v_box.pack_start(hBox_type, False, True, 0)
-
-        label_type = Gtk.Label()
-        label_type.set_text("Type: ")
-        hBox_type.pack_start(label_type, False, True, 0)
-
-        self.combobox = Gtk.ComboBox.new_with_model_and_entry(Gtk.ListStore(str, str))
-        self.combobox.set_entry_text_column(0)
-        hBox_type.pack_start(self.combobox, True, True, 0)
-
-
-        sw_hist_values = Gtk.ScrolledWindow()
-        v_box.pack_start(sw_hist_values, True, True, 0)
-
-        self.TreeViewChartValues = TreeViewChartValues()
-
-        sw_hist_values.add(self.TreeViewChartValues)
-
-        changed_cb = lambda *_: self.update_chart()
-        self.combobox.connect('changed',  changed_cb)
-        self.TreeViewChartValues.connect("values-changed", changed_cb)
-
-        self.update_chart()
-
-        from matplotlib.backends.backend_gtk3cairo import FigureCanvasGTK3Cairo as FigureCanvas
-        self.canvas = FigureCanvas(self.bar_chart)
-        self.add2(self.canvas)
-
-
-    def update_chart(self):
-        """Creates the chart from all values from TreeViewChartValues"""
-
-        import xsa.chart as chart
-        substrings = []
-        strings = []
-
-        it = self.combobox.get_active_iter()
-        if it is not None:
-            field = self.combobox.get_model()[it][1]
-            for r in self.results:
-                strings.append(r[field])
-        for sub in self.TreeViewChartValues.store:
-            if not sub[0] == self.TreeViewChartValues.inital_value:
-                substrings.append(sub[0])
-        data = chart.count_substrings(substrings, strings)
-
-        self.bar_chart = chart.create(data,len(self.results), len(self.TreeViewChartValues.store)-1, self.bar_chart)
-
-        if self.canvas is not None:
-            self.canvas.draw()
-
-    def update(self, data, root_type):
-        """
-        Updates chart and combobox of types for given Root-Type and results.
-        **Parameters**
-
-            :data: JsonTable, results of search
-            :root_type: str, Root-Type of search
-        """
-        self.results = data
-
-        import xsa.datatypereader as type_reader
-
-        store = Gtk.ListStore(str, str)
-        if data != []:
-            for key in data.headers():
-                label = type_reader.get_field_label_by_key(root_type, key)
-                store.append([label, key])
-
-        self.combobox.set_model(store)
-        self.combobox.set_active(0)
-        self.update_chart()
-
-
-class TreeViewChartValues(Gtk.TreeView, GObject.GObject):
-    """TreeView for creating the Search-Strings, shown in the chart."""
-
-    inital_value = "..."
-    """Start-string for new inserted row."""
-
-    tooltip = 'Delete by using right click'
-    """Tooltip that will be shown for each row and each column"""
-
-    __gsignals__= {'values-changed': (GObject.SIGNAL_RUN_FIRST, None,())}
-
-    def __init__(self):
-        """Creats the model and column."""
-
-        super(TreeViewChartValues, self).__init__()
-        self.create_model()
-        self.create_column()
-
-
-
-        def on_treeview_button_press_event(treeview, event):
-            if event.button == 3:
-                x = int(event.x)
-                y = int(event.y)
-                path_info = treeview.get_path_at_pos(x, y)
-                if path_info is not None:
-                    path, col, cellx, celly = path_info
-                    iter = self.store.get_iter(path)
-                    if self.store.get_value(iter,0) != self.inital_value:
-                        self.store.remove(iter)
-            self.emit("values-changed")
-
-        self.connect('button_press_event', on_treeview_button_press_event)
-        self.set_tooltip_column(1)
-
-
-    def create_column(self):
-        """Creates column and its edited-callback."""
-
-        def cell_edited_callback(cellrenderertext, path_string, new_text, *_):
-            it = self.store.get_iter_from_string(path_string)
-            it_last = self.store.get_iter(len(self.store)-1)
-            self.store.set(it, 0, new_text)
-            if not (it_last is None) and (self.store.get_value(it_last, 0) != self.inital_value):
-                self.store.append([