Merge branch 'release/release20150310' into develop

datatypes/ext_cSubAssessment.JSON

 {
 "root-type":"ext:cSubAssessment",
-"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/projects/{project}/experiments/{expt_id}"},
+"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}"},
 "fields":[
 {"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"},
 {"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"},

xsa/chart.py

@@ -30,26 +30,24 @@ def count_substrings(substrings, strings):
     return list(results.items())
 
 
-def create(file, data, size, y_max, x_max):
+def create(data, y_max, x_max, fig):
     """
     creates diagram for given 'data' and max-values ('y_max', 'x_max') and
     renders it as an image, saved as 'file' in given 'size'
 
     **Parameters**
-        :file: destination/filename where the rendered image will be saved
         :data: dictionary, the key will be shown on the x-axis. the value on the y-axis. if a two-dimensional list will be pass, the values on the first dimension will drawn on the x-axis, the second on the y-axis
-        :size: tuple (int,int), size of rendered image
         :y_max: max-value for y-axis
         :x_max: max-value for x-axis (= count of keys)
     """
-    import matplotlib.pyplot as plt
+
+    from matplotlib.figure import Figure
 
     GNOME_BLUE = '#3A81CC'  # the proper gtk-widgets-colour
     DPI = float(90)         # dots per inch
     BAR_WIDTH = 0.8         # width of bars
     BAR_START = 0.1         # start of bars
 
-    w, h = size
     labels = []
     values = []
     for lab, val in data:
@@ -62,24 +60,28 @@ def create(file, data, size, y_max, x_max):
         label_pos.append(x+BAR_START+BAR_WIDTH/2)
         x_pos.append(x+BAR_START)
 
-    _, ax = plt.subplots()
+    if fig is None:
+        fig=Figure()
+    fig.clear()
+
+    ax = fig.add_subplot(111)   #111: create a 1 x 1 grid, put the subplot in the 1st cell
 
     if y_max != 0:
-        plt.ylim((0,y_max))
-        plt.xlim((0,x_max))
+        ax.set_xlim(0, x_max)
+        ax.set_ylim(0, y_max)
     else:
-        plt.gca().axes.get_yaxis().set_visible(False)
+        ax.get_yaxis().set_visible(False)
 
-    bars = plt.bar(left=x_pos, height=values, width=BAR_WIDTH, color=GNOME_BLUE)
-    plt.xticks(label_pos, labels)
-    plt.ylabel('Count')
-    plt.gcf().set_size_inches(w/DPI,h/DPI)
+    bars = ax.bar(left=x_pos, height=values, width=BAR_WIDTH, color=GNOME_BLUE)
+
+    ax.get_xaxis().set_ticks(label_pos)
+    ax.get_xaxis().set_ticklabels(labels)
+    ax.get_yaxis().set_label_text('Count')
 
     for b in bars:
         height = b.get_height()
         ax.text(b.get_x()+b.get_width()/2., height, '%d'%int(height),
                 ha='center', va='bottom', color='black')
 
-
-    plt.savefig(file, dpi=DPI,transparent=False, pad_inches=0, frameon=None)
-    plt.close()
+    ax.plot()
+    return fig

xsa/errors.py

@@ -45,3 +45,11 @@ class NoComparisonDataError(Error):
 class CorruptedQueryError(Error):
     """Should be raised when the query-definition can not be loaded from file."""
     pass
+
+class DownloadError(Error):
+    """Should be raised when an error occurred while downloading."""
+    pass
+
+class WritingError(Error):
+    """Should be raised when an error occurred while writing something."""
+    pass

xsa/jsonutil.py

+"""
+:Author: Yannick Schwartz
+:Created: 2010/08/17
+
+copied from pyxnat: https://github.com/pyxnat
+"""
+
+
+import csv
+import copy
+from fnmatch import fnmatch
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+import json
+
+# jdata is a list of dicts
+
+def join_tables(join_column, jdata, *jtables):
+    indexes = []
+
+    for jtable in [jdata]+list(jtables):
+        if isinstance(jtable, dict):
+            jtable = [jtable]
+        index = {}
+        [index.setdefault(entry[join_column], entry) for entry in jtable]
+        indexes.append(index)
+
+    merged_jdata = []
+    for join_id in indexes[0].keys():
+        for index in indexes[1:]:
+            indexes[0][join_id].update(index[join_id])
+            merged_jdata.append(indexes[0][join_id])
+
+    return merged_jdata
+
+def get_column(jdata, col, val_pattern='*'):
+    if isinstance(jdata, dict):
+        jdata = [jdata]
+
+    if val_pattern == '*':
+        return [entry[col] for entry in jdata if entry.has_key(col)]
+    else:
+        return [entry[col] for entry in jdata
+                if fnmatch(entry.get(col), val_pattern)
+                ]
+
+def get_where(jdata, *args, **kwargs):
+    if isinstance(jdata, dict):
+        jdata = [jdata]
+
+    match = []
+
+    for entry in jdata:
+        match_args = all([arg in entry.keys() or arg in entry.values()
+                          for arg in args
+                          ])
+
+        match_kwargs = all([entry[key] == kwargs[key]
+                            for key in kwargs.keys()
+                            ])
+
+        if match_args and match_kwargs:
+            match.append(entry)
+
+    return match
+
+def get_where_not(jdata, *args, **kwargs):
+    if isinstance(jdata, dict):
+        jdata = [jdata]
+
+    match = []
+
+    for entry in jdata:
+        match_args = all([arg in entry.keys() or arg in entry.values()
+                          for arg in args
+                          ])
+
+        match_kwargs = all([entry[key] == kwargs[key]
+                            for key in kwargs.keys()
+                            ])
+
+        if not match_args and not match_kwargs:
+            match.append(entry)
+
+    return match
+
+def get_headers(jdata):
+    if isinstance(jdata, dict):
+        jdata = [jdata]
+    return [] if jdata == [] else jdata[0].keys()
+
+def get_selection(jdata, columns):
+    if isinstance(jdata, dict):
+        jdata = [jdata]
+
+    sub_table = copy.deepcopy(jdata)
+
+    rmcols = set(get_headers(jdata)).difference(columns)
+
+    for entry in sub_table:
+        for col in rmcols:
+            if entry.has_key(col):
+                del entry[col]
+
+    return sub_table
+
+def csv_to_json(csv_str):
+    csv_reader = csv.reader(StringIO(csv_str), delimiter=',', quotechar='"')
+    headers = csv_reader.next()
+
+    return [dict(zip(headers, entry)) for entry in csv_reader]
+
+
+class JsonTable(object):
+    """ Wrapper around a list of dictionnaries to provide utility functions.
+    """
+    def __init__(self, jdata, order_by=[]):
+        self.data = jdata
+        self.order_by = order_by
+
+    def __repr__(self):
+        # if len(self.data) == 0:
+        #     return '[]'
+        # elif len(self.data) == 1:
+        #     return str(self.data[0])
+
+        if len(self.headers()) <= 5:
+            _headers = ','.join(self.headers())
+        else:
+            _headers = '%s ... %s' % (','.join(self.headers()[:2]),
+                                      ','.join(self.headers()[-2:])
+                                      )
+
+        return '<JsonTable %s:%s> %s' % (
+            len(self), len(self.headers()), _headers
+            )
+
+        # return ('[%s\n .\n .\n . \n%s]\n\n'
+        #         '------------\n'
+        #         '%s rows\n'
+        #         '%s columns\n'
+        #         '%s characters') % (str(self.data[0]),
+        #                             str(self.data[-1]),
+        #                             len(self),
+        #                             len(self.headers()),
+        #                             len(self.dumps_csv())
+        #                             )
+
+    def __str__(self):
+        return self.dumps_csv()
+
+    def __len__(self):
+        return len(self.data)
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __getitem__(self, name):
+        if isinstance(name, (str, unicode)):
+            return self.get(name)
+        elif isinstance(name, int):
+            return self.__class__([self.data[name]], self.order_by)
+        elif isinstance(name, list):
+            return self.select(name)
+
+    def __getslice__(self, i, j):
+        return self.__class__(self.data[i:j], self.order_by)
+
+    def join(self, join_column, *jtables):
+        """ Join jsontables with a common column.
+
+            Parameters
+            ----------
+            join_column: string
+                The name or header of the join column.
+            jtables: *args
+                Other  jtables.
+        """
+        return self.__class__(
+            join_tables(join_column, self.data,
+                        *[jtable.data for jtable in jtables]),
+            self.order_by
+            )
+
+    def has_header(self, name):
+        return name in self.headers()
+
+    def headers(self):
+        """ Returns the headers of the object.
+        """
+        return get_headers(self.data)
+
+    def get(self, col, val_pattern='*', always_list=False):
+        """ Gets a single column value.
+
+            Parameters
+            ----------
+            col: string
+                The column name
+            val_pattern: string
+                Enable a filter on the values to be returned.
+            always_list: boolean
+                If only a single value is to be returned - i.e. there
+                is only on element in the list of dicts or there is only
+                one match against the value filter - is can be returned
+                within a list (with True) or not (default).
+        """
+        res = get_column(self.data, col, val_pattern)
+        if always_list:
+            return res
+        if len(self.data) == 1:
+            return res[0]
+        return res
+
+    def where(self, *args, **kwargs):
+        """ Filters the object.
+
+            Paramaters
+            ----------
+            args:
+                Value must be matched in the key or the value of an entry.
+            kwargs:
+                Value for a specific key must be matched in an entry.
+
+            Returns
+            -------
+            A :class:`JsonTable` containing the matches.
+        """
+        return self.__class__(get_where(self.data, *args, **kwargs),
+                              self.order_by
+                              )
+
+    def where_not(self, *args, **kwargs):
+        """ Filters the object. Conditions must not be matched.
+
+            Paramaters
+            ----------
+            args:
+                Value must not be matched in the key or the value of an
+                entry.
+            kwargs:
+                Value for a specific key must not be matched in an entry.
+
+            Returns
+            -------
+            A :class:`JsonTable` containing the not matches.
+        """
+        return self.__class__(get_where_not(self.data, *args, **kwargs),
+                              self.order_by
+                              )
+
+    def select(self, columns):
+        """ Select only some columns of interest.
+
+            Returns
+            -------
+            A :class:`JsonTable` with the selected columns.
+        """
+        return self.__class__(get_selection(self.data, columns),
+                              self.order_by
+                              )
+
+    def dump_csv(self, dest, delimiter=','):
+        """ Dumps the object content in a csv file format.
+
+            Parameters
+            ----------
+            dest: string
+                Destination file path.
+            delimiter: char
+                Character to separate values in the csv file.
+        """
+        fd = open(dest, 'w')
+        fd.write(self.dumps_csv(delimiter))
+        fd.close()
+
+    def dumps_csv(self, delimiter=','):
+        str_buffer = StringIO()
+        csv_writer = csv.writer(str_buffer, delimiter=delimiter,
+                                quotechar='"', quoting=csv.QUOTE_MINIMAL)
+
+        for entry in self.as_list():
+            csv_writer.writerow(entry)
+
+        return str_buffer.getvalue()
+
+    def dump_json(self, dest):
+        fd = open(dest, 'w')
+        fd.write(self.dumps_json())
+        fd.close()
+
+    def dumps_json(self):
+        return json.dumps(self.data)
+
+    def as_list(self):
+        table = [[]]
+
+        for header in self.order_by:
+            if header in self.headers():
+                table[0].append(header)
+        for header in self.headers():
+            if header not in self.order_by:
+                table[0].append(header)
+
+        for entry in self.data:
+            row = []
+            for header in self.order_by:
+                if entry.has_key(header):
+                    row.append(entry.get(header))
+            for header in self.headers():
+                if header not in self.order_by:
+                    row.append(entry.get(header))
+            table.append(row)
+
+        return table
+
+    def items(self):
+        table = []
+
+        for entry in self.data:
+            row = ()
+            for header in self.order_by:
+                if entry.has_key(header):
+                    row += (entry.get(header), )
+            for header in self.headers():
+                if header not in self.order_by:
+                    row += (entry.get(header), )
+            table.append(row)
+
+        return table

xsa/queries.py

@@ -34,7 +34,9 @@ a list of search-fields like:
 
 """
 
+
 from threading import Lock
+import xsa.errors
 requests_lock = Lock()
 
 def get_query_methods():
@@ -131,64 +133,58 @@ def download_file(url, creds, path):
                 handle.write(block)
         return True
     except IOError as e:
-        print "Error writing file %s, %s" % (path, e)
+        raise xsa.errors.WritingError("Error writing file %s, %s" % (path, e))
     except ValueError as e:
-        print "Error downloading file %s, Status Code %s" % (url, e)
+        raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
     return False
 
-
-def get_xnat_server_connection(force=False, host="", user="", passw=""):
-    from pyxnat import Interface
-    from tempfile import mkdtemp
-    import xsa.errors
-
-    if get_xnat_server_connection.cache and not force:
-        return get_xnat_server_connection.cache
-    elif force and host=="":
-        raise xsa.errors.ServerNotFoundError("The host was not defined.")
-    elif force and (user=="" or passw==""):
-        raise xsa.errors.UnauthorizedError("The user-name and/or the user-password were not defined.")
-
-    disconnect_xnat_server()
-    try:
-        get_xnat_server_connection.cache = Interface(server=host, user=user, password=passw, cachedir=mkdtemp())
-    except IndexError as e:
-        raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
-    return get_xnat_server_connection()
-get_xnat_server_connection.cache = None
-
-def disconnect_xnat_server():
-    import xsa.errors
-    try:
-        get_xnat_server_connection.cache.disconnect()
-        get_xnat_server_connection.cache = None
-    except:
-       return
-
+def retry_qry(fun):
+    def retry(*args, **kwargs):
+        last_exception = None
+        retries = 1
+        while retries >= 0:
+            try:
+                return fun(*args, **kwargs)
+            except Exception as e:
+                print(("wrapped", e))
+                if type(last_exception) is type(e):
+                    raise e
+                last_exception = e
+            finally:
+                retries -= 1
+    return retry
+
+@retry_qry
 def query_for_additional_data(rest, result, host, creds):
 
+    import requests
+    from requests.auth import HTTPBasicAuth
+    from xml.etree import ElementTree
     import re
+
     def subfunc(f):
         key = f.group(0).strip("{}")
         return result[key]
     ret = re.sub('\{\w+\}', subfunc, rest)
-
     user, passw = creds
-    central = get_xnat_server_connection(host=host, user=user, passw=passw)
-    exp = central.select(ret)
+    url = "%s%s" % (host, ret)
+
+    r = requests.get(url, auth=HTTPBasicAuth(user, passw))
+    exp = ElementTree.fromstring(r.text)
 
-    new_fields = exp.xpath("//xnat:field")
+    new_fields = exp.findall(".//xnat:field", {"xnat": "http://nrg.wustl.edu/xnat"})
     new_result = result.copy()
     for f in new_fields:
         fieldname = f.get("name")
-        value = exp.xpath("//xnat:field[@name='"+fieldname+"']/child::text()")
-        new_result[fieldname] = ''.join(value).replace("\n","")
+        value = exp.find(".//xnat:field[@name='"+fieldname+"']", {"xnat": "http://nrg.wustl.edu/xnat"})
+        new_result[fieldname] = ''.join(value.text).replace("\n","")
     return new_result
 
 
+@retry_qry
 def search_for(host, root_element, constraints, search_fields, user, passw):
     """
-    Does a search for given values. raises xsa -Exceptions
+    Does a search for given values. raises xsa-Exceptions
 
     **Parameters**
         :host: str, host-address
@@ -204,40 +200,62 @@ def search_for(host, root_element, constraints, search_fields, user, passw):
         :xsa.errors.ResponseNotReady:
         :xsa.errors.QueryError:
 
-
     """
-    import xsa.errors
-
-    from pyxnat.core import errors
-    from httplib2 import ServerNotFoundError
-    from httplib import ResponseNotReady
-
-    from tempfile import mkdtemp
-    tmp_dir=mkdtemp()
-
-    central = get_xnat_server_connection(host=host, user=user, passw=passw)
-
-    result = []
-
-    try:
-        result =  central.select(root_element,search_fields).where(constraints)
-    except errors.DatabaseError as e:
-        if '401' in str(e):
-            raise xsa.errors.UnauthorizedError("Unauthorizied attempt. Check your User and Password")
-        else:
-            raise e
-    except ServerNotFoundError:
-        raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
-    except ResponseNotReady:
-        raise xsa.errors.ResponseNotReady("Please check your Host-Address")
-
-    if result == []:
-        raise xsa.errors.QueryError("Please check your query.")
-
-    try:
-        from shutil import rmtree
-        rmtree(tmp_dir)
-    except:
-        pass
-
-    return result
+    import requests
+    from requests.auth import HTTPBasicAuth
+    from tempfile import NamedTemporaryFile
+    from xsa.jsonutil import JsonTable
+    import json
+
+    url= "%s%s" % (host, "/data/search?format=json")
+
+    with NamedTemporaryFile(suffix=".xml") as tmp:
+        tree = create_xml_tree(root_element, constraints, search_fields)
+        tree.write(tmp, xml_declaration=True)
+        tmp.flush()
+        r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))
+
+    if r.status_code == requests.codes.ok:
+        json_file=(json.loads(r.text))['ResultSet']['Result']
+        return JsonTable(json_file)
+
+
+def create_xml_tree(root_type, query, fields):
+    import xml.etree.cElementTree as ET
+
+    search_attributes = {"ID":"", "allow-diff-columns":"0", "secure":"false", "brief-description":"MR Sessions", "xmlns:xdat":"http://nrg.wustl.edu/security", "xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance"}
+    root = ET.Element("xdat:search", attrib=search_attributes)
+    ET.SubElement(root, "xdat:root_element_name").text = root_type
+
+    for field in fields:
+        element, field_id = field.split("/")
+        field = ET.SubElement(root, "xdat:search_field")
+        ET.SubElement(field, "xdat:element_name").text = element
+        ET.SubElement(field, "xdat:field_ID").text = field_id
+
+    def create_criterias(parent, element, iterator):
+        try:
+            if isinstance(element, str):
+                element = parent.set("method", element)
+                create_criterias(parent, next(iterator), iterator)
+            elif isinstance(element, tuple):
+                if len(element) != 3:
+                    raise errors.QueryError("The query-definition is incorrect.")
+                else:
+                    citeria = ET.SubElement(parent, "xdat:criteria", attrib={"override_value_formatting":"0"})
+                    ET.SubElement(citeria, "xdat:schema_field").text=element[0]
+                    ET.SubElement(citeria, "xdat:comparison_type").text=element[1]
+                    ET.SubElement(citeria