queries.py 8.49 KB
Newer Older
Franziska Koehn's avatar
Franziska Koehn committed
1
2
3
4
5
"""
:Author: Franziska Koehn
:Created: 2015/01/13

This module includes functions around sending and defining queries.
Franziska Koehn's avatar
Franziska Koehn committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

to query the xnat server define:

one string including one Root-Type like:

.. sourcecode:: python

   'xnat:mrScanData'

one list including the search-constraints like:

.. code-block:: python

    [
        ('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
        ('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
        'AND',
        [('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
         ('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
         'OR'
         ]
    ]

a list of search-fields like:

.. code-block:: python

    [ 'xnat:mrScanData/TYPE','xnat:mrScanData/ID']

Franziska Koehn's avatar
Franziska Koehn committed
35
36
"""

Franziska Koehn's avatar
Franziska Koehn committed
37

Franziska Koehn's avatar
Franziska Koehn committed
38
from threading import Lock
Franziska Koehn's avatar
Franziska Koehn committed
39
import xsa.errors
Franziska Koehn's avatar
Franziska Koehn committed
40
41
requests_lock = Lock()

42
def get_query_methods():
Franziska Koehn's avatar
Franziska Koehn committed
43
    """Returns all applicable methods for creating a query."""
44
45
46
    return ["AND", "OR"]

def get_operators():
Franziska Koehn's avatar
Franziska Koehn committed
47
    """Returns all applicable Operators for creating a query."""
48
49
    return ["LIKE", ">", "<", "<=", ">=", "="]

Franziska Koehn's avatar
Franziska Koehn committed
50

51
def download_async(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
52
    """
Franziska Koehn's avatar
Franziska Koehn committed
53
    Downloads file by using threads (So the GUI will not be blocked while downloading).
Franziska Koehn's avatar
Franziska Koehn committed
54
55

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
56
57
58
59
60
61
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
62
63
        :cb_args: args for function cb
    """
Franziska Koehn's avatar
Franziska Koehn committed
64
    from threading import Thread
65
    download_thread = Thread(target=download, args=(result, host, creds, rest, dest_folder, cb, cb_args))
Franziska Koehn's avatar
Franziska Koehn committed
66
67
    download_thread.start()
    return download_thread
Franziska Koehn's avatar
Franziska Koehn committed
68

69
def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
70
    """
Franziska Koehn's avatar
Franziska Koehn committed
71
    Downloads a Result.
Franziska Koehn's avatar
Franziska Koehn committed
72
73

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
74
75
76
77
78
79
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
80
81
82
        :cb_args: args for function cb
    """

Franziska Koehn's avatar
Franziska Koehn committed
83
    requests_lock.acquire()
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

    import re
    import os

    names=[]
    def subfunc(f):
        key = f.group(0).strip("{}")
        r = result[key]
        names.append(r)
        return r
    ret = re.sub('\{\w+\}', subfunc, rest)

    path = os.path.join(dest_folder, '-'.join(names))
    url = "%s%s" % (host, ret)

    download_file(url, creds, path)
Franziska Koehn's avatar
Franziska Koehn committed
100
101
    requests_lock.release()
    cb(*cb_args)
102

103
def download_file(url, creds, path):
Franziska Koehn's avatar
Franziska Koehn committed
104
    """
Franziska Koehn's avatar
Franziska Koehn committed
105
106
107
108
    Downloads a file from given 'url' and saves it to 'path'.

    Returns True if download was finished without problems
    Returns False if download raised an exception.
Franziska Koehn's avatar
Franziska Koehn committed
109
110

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
111
112
113
        :url: str, host/REST-API with filled values
        :creds: tuple(user-name, user-password), credentials
        :path: str, folder where the downloaded files will be saved
Franziska Koehn's avatar
Franziska Koehn committed
114
115
    """

Franziska Koehn's avatar
Franziska Koehn committed
116
117
118
119
    import requests
    from base64 import b64encode
    from requests.auth import HTTPBasicAuth

120
121
    if not path.endswith(".zip"):
        path += ".zip"
Franziska Koehn's avatar
Franziska Koehn committed
122

123
    user, passw = creds
Franziska Koehn's avatar
Franziska Koehn committed
124
125

    try:
126
        with open(path, 'wb') as handle:
Franziska Koehn's avatar
Franziska Koehn committed
127
128
129
130
131
132
133
            response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
            if not response.ok:
                raise ValueError(response.status_code)
            for block in response.iter_content(1024):
                if not block:
                    break
                handle.write(block)
Franziska Koehn's avatar
Franziska Koehn committed
134
        return True
Franziska Koehn's avatar
Franziska Koehn committed
135
    except IOError as e:
Franziska Koehn's avatar
Franziska Koehn committed
136
        raise xsa.errors.WritingError("Error writing file %s, %s" % (path, e))
Franziska Koehn's avatar
Franziska Koehn committed
137
    except ValueError as e:
Franziska Koehn's avatar
Franziska Koehn committed
138
        raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
Franziska Koehn's avatar
Franziska Koehn committed
139
    return False
Franziska Koehn's avatar
Franziska Koehn committed
140

141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def retry_qry(fun):
    def retry(*args, **kwargs):
        last_exception = None
        retries = 1
        while retries >= 0:
            try:
                return fun(*args, **kwargs)
            except Exception as e:
                print(("wrapped", e))
                if type(last_exception) is type(e):
                    raise e
                last_exception = e
            finally:
                retries -= 1
    return retry

@retry_qry
158
def query_for_additional_data(rest, result, host, creds):
159

Franziska Koehn's avatar
Franziska Koehn committed
160
161
162
    import requests
    from requests.auth import HTTPBasicAuth
    from xml.etree import ElementTree
163
    import re
Franziska Koehn's avatar
Franziska Koehn committed
164

165
166
    def subfunc(f):
        key = f.group(0).strip("{}")
Franziska Koehn's avatar
Franziska Koehn committed
167
        return result[key]
168
    ret = re.sub('\{\w+\}', subfunc, rest)
169
    user, passw = creds
Franziska Koehn's avatar
Franziska Koehn committed
170
171
172
173
    url = "%s%s" % (host, ret)

    r = requests.get(url, auth=HTTPBasicAuth(user, passw))
    exp = ElementTree.fromstring(r.text)
174

Franziska Koehn's avatar
Franziska Koehn committed
175
    new_fields = exp.findall(".//xnat:field", {"xnat": "http://nrg.wustl.edu/xnat"})
176
177
178
    new_result = result.copy()
    for f in new_fields:
        fieldname = f.get("name")
Franziska Koehn's avatar
Franziska Koehn committed
179
180
        value = exp.find(".//xnat:field[@name='"+fieldname+"']", {"xnat": "http://nrg.wustl.edu/xnat"})
        new_result[fieldname] = ''.join(value.text).replace("\n","")
181
182
    return new_result

Franziska Koehn's avatar
Franziska Koehn committed
183

184
@retry_qry
185
def search_for(host, root_element, constraints, search_fields, user, passw):
Franziska Koehn's avatar
Franziska Koehn committed
186
    """
Franziska Koehn's avatar
Franziska Koehn committed
187
    Does a search for given values. raises xsa-Exceptions
Franziska Koehn's avatar
Franziska Koehn committed
188
189

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
190
191
192
193
194
195
196
197
198
199
200
201
202
        :host: str, host-address
        :root_element: str, Root-Type of search
        :constraints: list, constraints of query
        :search_fields: list of str, fields which will be returned from server
        :user: str, user-name
        :passw: str, user-password

    **Raises**
        :xsa.errors.ServerNotFoundError:
        :xsa.errors.UnauthorizedError:
        :xsa.errors.ResponseNotReady:
        :xsa.errors.QueryError:

Franziska Koehn's avatar
Franziska Koehn committed
203
    """
Franziska Koehn's avatar
Franziska Koehn committed
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
    import requests
    from requests.auth import HTTPBasicAuth
    from tempfile import NamedTemporaryFile
    from xsa.jsonutil import JsonTable
    import json

    url= "%s%s" % (host, "/data/search?format=json")

    with NamedTemporaryFile(suffix=".xml") as tmp:
        tree = create_xml_tree(root_element, constraints, search_fields)
        tree.write(tmp, xml_declaration=True)
        tmp.flush()
        r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))

    if r.status_code == requests.codes.ok:
        json_file=(json.loads(r.text))['ResultSet']['Result']
        return JsonTable(json_file)


def create_xml_tree(root_type, query, fields):
    import xml.etree.cElementTree as ET

    search_attributes = {"ID":"", "allow-diff-columns":"0", "secure":"false", "brief-description":"MR Sessions", "xmlns:xdat":"http://nrg.wustl.edu/security", "xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance"}
    root = ET.Element("xdat:search", attrib=search_attributes)
    ET.SubElement(root, "xdat:root_element_name").text = root_type

    for field in fields:
        element, field_id = field.split("/")
        field = ET.SubElement(root, "xdat:search_field")
        ET.SubElement(field, "xdat:element_name").text = element
        ET.SubElement(field, "xdat:field_ID").text = field_id

    def create_criterias(parent, element, iterator):
        try:
            if isinstance(element, str):
                element = parent.set("method", element)
                create_criterias(parent, next(iterator), iterator)
            elif isinstance(element, tuple):
                if len(element) != 3:
                    raise errors.QueryError("The query-definition is incorrect.")
                else:
                    citeria = ET.SubElement(parent, "xdat:criteria", attrib={"override_value_formatting":"0"})
                    ET.SubElement(citeria, "xdat:schema_field").text=element[0]
                    ET.SubElement(citeria, "xdat:comparison_type").text=element[1]
                    ET.SubElement(citeria, "value").text=element[2]
                    create_criterias(parent, next(iterator), iterator)
            elif isinstance(element, list):
                child = ET.SubElement(parent, "xdat:child_set", method="AND")
                iter_child = iter(element)
                create_criterias(child, next(iter_child), iter_child)
        except StopIteration:
            pass

    iter_query = iter(query)
    where = ET.SubElement(root, "xdat:search_where", method="AND")
    create_criterias(where, next(iter_query), iter_query)

    return ET.ElementTree(root)