Commit c64c91a0 authored by Franziska Koehn's avatar Franziska Koehn
Browse files

Merge branch 'release/release20150310' into develop

parents 07f67d22 305a2c9b
{
"root-type":"ext:cSubAssessment",
"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/projects/{project}/experiments/{expt_id}"},
"comparison_data":{"identifier":["expt_id", "project"],"extra-source":"/data/archive/projects/{project}/experiments/{expt_id}"},
"fields":[
{"label":"Subject-ID", "field":"ext:cSubAssessment/SUBJECT_ID", "required":false, "key": "subject_id"},
{"label":"Experiment-ID", "field":"ext:cSubAssessment/EXPT_ID", "required":true, "key": "expt_id"},
......
......@@ -30,26 +30,24 @@ def count_substrings(substrings, strings):
return list(results.items())
def create(file, data, size, y_max, x_max):
def create(data, y_max, x_max, fig):
"""
creates diagram for given 'data' and max-values ('y_max', 'x_max') and
renders it as an image, saved as 'file' in given 'size'
**Parameters**
:file: destination/filename where the rendered image will be saved
:data: dictionary, the key will be shown on the x-axis. the value on the y-axis. if a two-dimensional list will be pass, the values on the first dimension will drawn on the x-axis, the second on the y-axis
:size: tuple (int,int), size of rendered image
:y_max: max-value for y-axis
:x_max: max-value for x-axis (= count of keys)
"""
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
GNOME_BLUE = '#3A81CC' # the proper gtk-widgets-colour
DPI = float(90) # dots per inch
BAR_WIDTH = 0.8 # width of bars
BAR_START = 0.1 # start of bars
w, h = size
labels = []
values = []
for lab, val in data:
......@@ -62,24 +60,28 @@ def create(file, data, size, y_max, x_max):
label_pos.append(x+BAR_START+BAR_WIDTH/2)
x_pos.append(x+BAR_START)
_, ax = plt.subplots()
if fig is None:
fig=Figure()
fig.clear()
ax = fig.add_subplot(111) #111: create a 1 x 1 grid, put the subplot in the 1st cell
if y_max != 0:
plt.ylim((0,y_max))
plt.xlim((0,x_max))
ax.set_xlim(0, x_max)
ax.set_ylim(0, y_max)
else:
plt.gca().axes.get_yaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
bars = plt.bar(left=x_pos, height=values, width=BAR_WIDTH, color=GNOME_BLUE)
plt.xticks(label_pos, labels)
plt.ylabel('Count')
plt.gcf().set_size_inches(w/DPI,h/DPI)
bars = ax.bar(left=x_pos, height=values, width=BAR_WIDTH, color=GNOME_BLUE)
ax.get_xaxis().set_ticks(label_pos)
ax.get_xaxis().set_ticklabels(labels)
ax.get_yaxis().set_label_text('Count')
for b in bars:
height = b.get_height()
ax.text(b.get_x()+b.get_width()/2., height, '%d'%int(height),
ha='center', va='bottom', color='black')
plt.savefig(file, dpi=DPI,transparent=False, pad_inches=0, frameon=None)
plt.close()
ax.plot()
return fig
......@@ -45,3 +45,11 @@ class NoComparisonDataError(Error):
class CorruptedQueryError(Error):
"""Should be raised when the query-definition can not be loaded from file."""
pass
class DownloadError(Error):
"""Should be raised when an error occurred while downloading."""
pass
class WritingError(Error):
"""Should be raised when an error occurred while writing something."""
pass
"""
:Author: Yannick Schwartz
:Created: 2010/08/17
copied from pyxnat: https://github.com/pyxnat
"""
import csv
import copy
from fnmatch import fnmatch
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
import json
# jdata is a list of dicts
def join_tables(join_column, jdata, *jtables):
indexes = []
for jtable in [jdata]+list(jtables):
if isinstance(jtable, dict):
jtable = [jtable]
index = {}
[index.setdefault(entry[join_column], entry) for entry in jtable]
indexes.append(index)
merged_jdata = []
for join_id in indexes[0].keys():
for index in indexes[1:]:
indexes[0][join_id].update(index[join_id])
merged_jdata.append(indexes[0][join_id])
return merged_jdata
def get_column(jdata, col, val_pattern='*'):
if isinstance(jdata, dict):
jdata = [jdata]
if val_pattern == '*':
return [entry[col] for entry in jdata if entry.has_key(col)]
else:
return [entry[col] for entry in jdata
if fnmatch(entry.get(col), val_pattern)
]
def get_where(jdata, *args, **kwargs):
if isinstance(jdata, dict):
jdata = [jdata]
match = []
for entry in jdata:
match_args = all([arg in entry.keys() or arg in entry.values()
for arg in args
])
match_kwargs = all([entry[key] == kwargs[key]
for key in kwargs.keys()
])
if match_args and match_kwargs:
match.append(entry)
return match
def get_where_not(jdata, *args, **kwargs):
if isinstance(jdata, dict):
jdata = [jdata]
match = []
for entry in jdata:
match_args = all([arg in entry.keys() or arg in entry.values()
for arg in args
])
match_kwargs = all([entry[key] == kwargs[key]
for key in kwargs.keys()
])
if not match_args and not match_kwargs:
match.append(entry)
return match
def get_headers(jdata):
if isinstance(jdata, dict):
jdata = [jdata]
return [] if jdata == [] else jdata[0].keys()
def get_selection(jdata, columns):
if isinstance(jdata, dict):
jdata = [jdata]
sub_table = copy.deepcopy(jdata)
rmcols = set(get_headers(jdata)).difference(columns)
for entry in sub_table:
for col in rmcols:
if entry.has_key(col):
del entry[col]
return sub_table
def csv_to_json(csv_str):
csv_reader = csv.reader(StringIO(csv_str), delimiter=',', quotechar='"')
headers = csv_reader.next()
return [dict(zip(headers, entry)) for entry in csv_reader]
class JsonTable(object):
""" Wrapper around a list of dictionnaries to provide utility functions.
"""
def __init__(self, jdata, order_by=[]):
self.data = jdata
self.order_by = order_by
def __repr__(self):
# if len(self.data) == 0:
# return '[]'
# elif len(self.data) == 1:
# return str(self.data[0])
if len(self.headers()) <= 5:
_headers = ','.join(self.headers())
else:
_headers = '%s ... %s' % (','.join(self.headers()[:2]),
','.join(self.headers()[-2:])
)
return '<JsonTable %s:%s> %s' % (
len(self), len(self.headers()), _headers
)
# return ('[%s\n .\n .\n . \n%s]\n\n'
# '------------\n'
# '%s rows\n'
# '%s columns\n'
# '%s characters') % (str(self.data[0]),
# str(self.data[-1]),
# len(self),
# len(self.headers()),
# len(self.dumps_csv())
# )
def __str__(self):
return self.dumps_csv()
def __len__(self):
return len(self.data)
def __iter__(self):
return iter(self.data)
def __getitem__(self, name):
if isinstance(name, (str, unicode)):
return self.get(name)
elif isinstance(name, int):
return self.__class__([self.data[name]], self.order_by)
elif isinstance(name, list):
return self.select(name)
def __getslice__(self, i, j):
return self.__class__(self.data[i:j], self.order_by)
def join(self, join_column, *jtables):
""" Join jsontables with a common column.
Parameters
----------
join_column: string
The name or header of the join column.
jtables: *args
Other jtables.
"""
return self.__class__(
join_tables(join_column, self.data,
*[jtable.data for jtable in jtables]),
self.order_by
)
def has_header(self, name):
return name in self.headers()
def headers(self):
""" Returns the headers of the object.
"""
return get_headers(self.data)
def get(self, col, val_pattern='*', always_list=False):
""" Gets a single column value.
Parameters
----------
col: string
The column name
val_pattern: string
Enable a filter on the values to be returned.
always_list: boolean
If only a single value is to be returned - i.e. there
is only on element in the list of dicts or there is only
one match against the value filter - is can be returned
within a list (with True) or not (default).
"""
res = get_column(self.data, col, val_pattern)
if always_list:
return res
if len(self.data) == 1:
return res[0]
return res
def where(self, *args, **kwargs):
""" Filters the object.
Paramaters
----------
args:
Value must be matched in the key or the value of an entry.
kwargs:
Value for a specific key must be matched in an entry.
Returns
-------
A :class:`JsonTable` containing the matches.
"""
return self.__class__(get_where(self.data, *args, **kwargs),
self.order_by
)
def where_not(self, *args, **kwargs):
""" Filters the object. Conditions must not be matched.
Paramaters
----------
args:
Value must not be matched in the key or the value of an
entry.
kwargs:
Value for a specific key must not be matched in an entry.
Returns
-------
A :class:`JsonTable` containing the not matches.
"""
return self.__class__(get_where_not(self.data, *args, **kwargs),
self.order_by
)
def select(self, columns):
""" Select only some columns of interest.
Returns
-------
A :class:`JsonTable` with the selected columns.
"""
return self.__class__(get_selection(self.data, columns),
self.order_by
)
def dump_csv(self, dest, delimiter=','):
""" Dumps the object content in a csv file format.
Parameters
----------
dest: string
Destination file path.
delimiter: char
Character to separate values in the csv file.
"""
fd = open(dest, 'w')
fd.write(self.dumps_csv(delimiter))
fd.close()
def dumps_csv(self, delimiter=','):
str_buffer = StringIO()
csv_writer = csv.writer(str_buffer, delimiter=delimiter,
quotechar='"', quoting=csv.QUOTE_MINIMAL)
for entry in self.as_list():
csv_writer.writerow(entry)
return str_buffer.getvalue()
def dump_json(self, dest):
fd = open(dest, 'w')
fd.write(self.dumps_json())
fd.close()
def dumps_json(self):
return json.dumps(self.data)
def as_list(self):
table = [[]]
for header in self.order_by:
if header in self.headers():
table[0].append(header)
for header in self.headers():
if header not in self.order_by:
table[0].append(header)
for entry in self.data:
row = []
for header in self.order_by:
if entry.has_key(header):
row.append(entry.get(header))
for header in self.headers():
if header not in self.order_by:
row.append(entry.get(header))
table.append(row)
return table
def items(self):
table = []
for entry in self.data:
row = ()
for header in self.order_by:
if entry.has_key(header):
row += (entry.get(header), )
for header in self.headers():
if header not in self.order_by:
row += (entry.get(header), )
table.append(row)
return table
......@@ -34,7 +34,9 @@ a list of search-fields like:
"""
from threading import Lock
import xsa.errors
requests_lock = Lock()
def get_query_methods():
......@@ -131,64 +133,58 @@ def download_file(url, creds, path):
handle.write(block)
return True
except IOError as e:
print "Error writing file %s, %s" % (path, e)
raise xsa.errors.WritingError("Error writing file %s, %s" % (path, e))
except ValueError as e:
print "Error downloading file %s, Status Code %s" % (url, e)
raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
return False
def get_xnat_server_connection(force=False, host="", user="", passw=""):
from pyxnat import Interface
from tempfile import mkdtemp
import xsa.errors
if get_xnat_server_connection.cache and not force:
return get_xnat_server_connection.cache
elif force and host=="":
raise xsa.errors.ServerNotFoundError("The host was not defined.")
elif force and (user=="" or passw==""):
raise xsa.errors.UnauthorizedError("The user-name and/or the user-password were not defined.")
disconnect_xnat_server()
try:
get_xnat_server_connection.cache = Interface(server=host, user=user, password=passw, cachedir=mkdtemp())
except IndexError as e:
raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
return get_xnat_server_connection()
get_xnat_server_connection.cache = None
def disconnect_xnat_server():
import xsa.errors
try:
get_xnat_server_connection.cache.disconnect()
get_xnat_server_connection.cache = None
except:
return
def retry_qry(fun):
def retry(*args, **kwargs):
last_exception = None
retries = 1
while retries >= 0:
try:
return fun(*args, **kwargs)
except Exception as e:
print(("wrapped", e))
if type(last_exception) is type(e):
raise e
last_exception = e
finally:
retries -= 1
return retry
@retry_qry
def query_for_additional_data(rest, result, host, creds):
import requests
from requests.auth import HTTPBasicAuth
from xml.etree import ElementTree
import re
def subfunc(f):
key = f.group(0).strip("{}")
return result[key]
ret = re.sub('\{\w+\}', subfunc, rest)
user, passw = creds
central = get_xnat_server_connection(host=host, user=user, passw=passw)
exp = central.select(ret)
url = "%s%s" % (host, ret)
r = requests.get(url, auth=HTTPBasicAuth(user, passw))
exp = ElementTree.fromstring(r.text)
new_fields = exp.xpath("//xnat:field")
new_fields = exp.findall(".//xnat:field", {"xnat": "http://nrg.wustl.edu/xnat"})
new_result = result.copy()
for f in new_fields:
fieldname = f.get("name")
value = exp.xpath("//xnat:field[@name='"+fieldname+"']/child::text()")
new_result[fieldname] = ''.join(value).replace("\n","")
value = exp.find(".//xnat:field[@name='"+fieldname+"']", {"xnat": "http://nrg.wustl.edu/xnat"})
new_result[fieldname] = ''.join(value.text).replace("\n","")
return new_result
@retry_qry
def search_for(host, root_element, constraints, search_fields, user, passw):
"""
Does a search for given values. raises xsa -Exceptions
Does a search for given values. raises xsa-Exceptions
**Parameters**
:host: str, host-address
......@@ -204,40 +200,62 @@ def search_for(host, root_element, constraints, search_fields, user, passw):
:xsa.errors.ResponseNotReady:
:xsa.errors.QueryError:
"""
import xsa.errors
from pyxnat.core import errors
from httplib2 import ServerNotFoundError
from httplib import ResponseNotReady
from tempfile import mkdtemp
tmp_dir=mkdtemp()
central = get_xnat_server_connection(host=host, user=user, passw=passw)
result = []
try:
result = central.select(root_element,search_fields).where(constraints)
except errors.DatabaseError as e:
if '401' in str(e):
raise xsa.errors.UnauthorizedError("Unauthorizied attempt. Check your User and Password")
else:
raise e
except ServerNotFoundError:
raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
except ResponseNotReady:
raise xsa.errors.ResponseNotReady("Please check your Host-Address")
if result == []:
raise xsa.errors.QueryError("Please check your query.")
try:
from shutil import rmtree
rmtree(tmp_dir)
except:
pass
return result
import requests
from requests.auth import HTTPBasicAuth
from tempfile import NamedTemporaryFile
from xsa.jsonutil import JsonTable
import json
url= "%s%s" % (host, "/data/search?format=json")
with NamedTemporaryFile(suffix=".xml") as tmp:
tree = create_xml_tree(root_element, constraints, search_fields)
tree.write(tmp, xml_declaration=True)
tmp.flush()
r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))
if r.status_code == requests.codes.ok:
json_file=(json.loads(r.text))['ResultSet']['Result']
return JsonTable(json_file)
def create_xml_tree(root_type, query, fields):
import xml.etree.cElementTree as ET
search_attributes = {"ID":"", "allow-diff-columns":"0", "secure":"false", "brief-description":"MR Sessions", "xmlns:xdat":"http://nrg.wustl.edu/security", "xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance"}
root = ET.Element("xdat:search", attrib=search_attributes)
ET.SubElement(root, "xdat:root_element_name").text = root_type
for field in fields:
element, field_id = field.split("/")
field = ET.SubElement(root, "xdat:search_field")
ET.SubElement(field, "xdat:element_name").text = element
ET.SubElement(field, "xdat:field_ID").text = field_id
def create_criterias(parent, element, iterator):
try:
if isinstance(element, str):
element = parent.set("method", element)
create_criterias(parent, next(iterator), iterator)
elif isinstance(element, tuple):
if len(element) != 3:
raise errors.QueryError("The query-definition is incorrect.")
else:
citeria = ET.SubElement(parent, "xdat:criteria", attrib={"override_value_formatting":"0"})
ET.SubElement(citeria, "xdat:schema_field").text=element[0]
ET.SubElement(citeria, "xdat:comparison_type").text=element[1]
ET.SubElement(citeria