queries.py 9.6 KB
Newer Older
Franziska Koehn's avatar
Franziska Koehn committed
1
2
3
4
5
"""
:Author: Franziska Koehn
:Created: 2015/01/13

This module includes functions around sending and defining queries.
Franziska Koehn's avatar
Franziska Koehn committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

to query the xnat server define:

one string including one Root-Type like:

.. sourcecode:: python

   'xnat:mrScanData'

one list including the search-constraints like:

.. code-block:: python

    [
        ('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
        ('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
        'AND',
        [('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
         ('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
         'OR'
         ]
    ]

a list of search-fields like:

.. code-block:: python

    [ 'xnat:mrScanData/TYPE','xnat:mrScanData/ID']

Franziska Koehn's avatar
Franziska Koehn committed
35
36
"""

Franziska Koehn's avatar
Franziska Koehn committed
37

Franziska Koehn's avatar
Franziska Koehn committed
38
from threading import Lock
Franziska Koehn's avatar
Franziska Koehn committed
39
import xsa.errors
Franziska Koehn's avatar
Franziska Koehn committed
40
41
requests_lock = Lock()

42
def get_query_methods():
Franziska Koehn's avatar
Franziska Koehn committed
43
    """Returns all applicable methods for creating a query."""
44
45
46
    return ["AND", "OR"]

def get_operators():
Franziska Koehn's avatar
Franziska Koehn committed
47
    """Returns all applicable Operators for creating a query."""
48
49
    return ["LIKE", ">", "<", "<=", ">=", "="]

Franziska Koehn's avatar
Franziska Koehn committed
50

51
def download_async(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
52
    """
Franziska Koehn's avatar
Franziska Koehn committed
53
    Downloads file by using threads (So the GUI will not be blocked while downloading).
Franziska Koehn's avatar
Franziska Koehn committed
54
55

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
56
57
58
59
60
61
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
62
63
        :cb_args: args for function cb
    """
Franziska Koehn's avatar
Franziska Koehn committed
64
    from threading import Thread
65
    download_thread = Thread(target=download, args=(result, host, creds, rest, dest_folder, cb, cb_args))
Franziska Koehn's avatar
Franziska Koehn committed
66
67
    download_thread.start()
    return download_thread
Franziska Koehn's avatar
Franziska Koehn committed
68

69
def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
70
    """
Franziska Koehn's avatar
Franziska Koehn committed
71
    Downloads a Result.
Franziska Koehn's avatar
Franziska Koehn committed
72
73

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
74
75
76
77
78
79
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
80
81
82
        :cb_args: args for function cb
    """

Franziska Koehn's avatar
Franziska Koehn committed
83
    requests_lock.acquire()
84
85
86
87
88
89
90
91
92
93
94
95
96

    import re
    import os

    names=[]
    def subfunc(f):
        key = f.group(0).strip("{}")
        r = result[key]
        names.append(r)
        return r
    ret = re.sub('\{\w+\}', subfunc, rest)

    path = os.path.join(dest_folder, '-'.join(names))
97
    url = "%s%s" % (host.strip("/"), ret)
98
99

    download_file(url, creds, path)
Franziska Koehn's avatar
Franziska Koehn committed
100
101
    requests_lock.release()
    cb(*cb_args)
102

103
def download_file(url, creds, path):
Franziska Koehn's avatar
Franziska Koehn committed
104
    """
Franziska Koehn's avatar
Franziska Koehn committed
105
106
107
108
    Downloads a file from given 'url' and saves it to 'path'.

    Returns True if download was finished without problems
    Returns False if download raised an exception.
Franziska Koehn's avatar
Franziska Koehn committed
109
110

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
111
112
113
        :url: str, host/REST-API with filled values
        :creds: tuple(user-name, user-password), credentials
        :path: str, folder where the downloaded files will be saved
Franziska Koehn's avatar
Franziska Koehn committed
114
115
    """

Franziska Koehn's avatar
Franziska Koehn committed
116
117
118
119
    import requests
    from base64 import b64encode
    from requests.auth import HTTPBasicAuth

120
121
    if not path.endswith(".zip"):
        path += ".zip"
Franziska Koehn's avatar
Franziska Koehn committed
122

123
    user, passw = creds
Franziska Koehn's avatar
Franziska Koehn committed
124
125

    try:
126
        with open(path, 'wb') as handle:
Franziska Koehn's avatar
Franziska Koehn committed
127
128
129
130
131
132
133
            response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
            if not response.ok:
                raise ValueError(response.status_code)
            for block in response.iter_content(1024):
                if not block:
                    break
                handle.write(block)
Franziska Koehn's avatar
Franziska Koehn committed
134
        return True
Franziska Koehn's avatar
Franziska Koehn committed
135
    except IOError as e:
Franziska Koehn's avatar
Franziska Koehn committed
136
        raise xsa.errors.WritingError("Error writing file %s, %s" % (path, e))
Franziska Koehn's avatar
Franziska Koehn committed
137
    except ValueError as e:
Franziska Koehn's avatar
Franziska Koehn committed
138
        raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))
Franziska Koehn's avatar
Franziska Koehn committed
139
    return False
Franziska Koehn's avatar
Franziska Koehn committed
140

141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
def download_file_iter(url, creds):
    """
    Downloads a file from given 'url'.

    Returns Iter of response
    Raises xsa.errors.WritingError and xsa.errors.DownloadError

    **Parameters**
        :url: str, host/REST-API with filled values
        :creds: tuple(user-name, user-password), credentials
    """

    import requests
    from base64 import b64encode
    from requests.auth import HTTPBasicAuth

    user, passw = creds

    try:
        response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
        if not response.ok:
            raise ValueError(response.status_code)
        return response.iter_content(1024)
    except IOError as e:
        raise xsa.errors.WritingError("Error writing file %s, %s" % (path, e))
    except ValueError as e:
        raise xsa.errors.DownloadError("Error downloading file %s, Status Code %s" % (url, e))

def prepare_rest(result, rest, host):
    import re

    def subfunc(f):
        key = f.group(0).strip("{}")
        return result[key]
    ret = re.sub('\{\w+\}', subfunc, rest)

    return "%s%s" % (host, ret)

179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def retry_qry(fun):
    def retry(*args, **kwargs):
        last_exception = None
        retries = 1
        while retries >= 0:
            try:
                return fun(*args, **kwargs)
            except Exception as e:
                print(("wrapped", e))
                if type(last_exception) is type(e):
                    raise e
                last_exception = e
            finally:
                retries -= 1
    return retry

@retry_qry
196
def query_for_additional_data(rest, result, host, creds):
197

Franziska Koehn's avatar
Franziska Koehn committed
198
199
200
    import requests
    from requests.auth import HTTPBasicAuth
    from xml.etree import ElementTree
201
    import re
Franziska Koehn's avatar
Franziska Koehn committed
202

203
204
    def subfunc(f):
        key = f.group(0).strip("{}")
Franziska Koehn's avatar
Franziska Koehn committed
205
        return result[key]
206
    ret = re.sub('\{\w+\}', subfunc, rest)
207
    user, passw = creds
208
    url = "%s%s" % (host.strip("/"), ret)
Franziska Koehn's avatar
Franziska Koehn committed
209
210
211

    r = requests.get(url, auth=HTTPBasicAuth(user, passw))
    exp = ElementTree.fromstring(r.text)
212

Franziska Koehn's avatar
Franziska Koehn committed
213
    new_fields = exp.findall(".//xnat:field", {"xnat": "http://nrg.wustl.edu/xnat"})
214
215
216
    new_result = result.copy()
    for f in new_fields:
        fieldname = f.get("name")
Franziska Koehn's avatar
Franziska Koehn committed
217
218
        value = exp.find(".//xnat:field[@name='"+fieldname+"']", {"xnat": "http://nrg.wustl.edu/xnat"})
        new_result[fieldname] = ''.join(value.text).replace("\n","")
219
220
    return new_result

Franziska Koehn's avatar
Franziska Koehn committed
221

222
@retry_qry
223
def search_for(host, root_element, constraints, search_fields, user, passw):
Franziska Koehn's avatar
Franziska Koehn committed
224
    """
Franziska Koehn's avatar
Franziska Koehn committed
225
    Does a search for given values. raises xsa-Exceptions
Franziska Koehn's avatar
Franziska Koehn committed
226
227

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
228
229
230
231
232
233
234
235
236
237
238
239
240
        :host: str, host-address
        :root_element: str, Root-Type of search
        :constraints: list, constraints of query
        :search_fields: list of str, fields which will be returned from server
        :user: str, user-name
        :passw: str, user-password

    **Raises**
        :xsa.errors.ServerNotFoundError:
        :xsa.errors.UnauthorizedError:
        :xsa.errors.ResponseNotReady:
        :xsa.errors.QueryError:

Franziska Koehn's avatar
Franziska Koehn committed
241
    """
Franziska Koehn's avatar
Franziska Koehn committed
242
243
244
245
246
247
    import requests
    from requests.auth import HTTPBasicAuth
    from tempfile import NamedTemporaryFile
    from xsa.jsonutil import JsonTable
    import json

248
    url = "%s%s" % (host.strip("/"), "/data/search?format=json")
Franziska Koehn's avatar
Franziska Koehn committed
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299

    with NamedTemporaryFile(suffix=".xml") as tmp:
        tree = create_xml_tree(root_element, constraints, search_fields)
        tree.write(tmp, xml_declaration=True)
        tmp.flush()
        r = requests.post(url, files={'file': open(tmp.name, 'rb')}, auth=HTTPBasicAuth(user, passw))

    if r.status_code == requests.codes.ok:
        json_file=(json.loads(r.text))['ResultSet']['Result']
        return JsonTable(json_file)


def create_xml_tree(root_type, query, fields):
    import xml.etree.cElementTree as ET

    search_attributes = {"ID":"", "allow-diff-columns":"0", "secure":"false", "brief-description":"MR Sessions", "xmlns:xdat":"http://nrg.wustl.edu/security", "xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance"}
    root = ET.Element("xdat:search", attrib=search_attributes)
    ET.SubElement(root, "xdat:root_element_name").text = root_type

    for field in fields:
        element, field_id = field.split("/")
        field = ET.SubElement(root, "xdat:search_field")
        ET.SubElement(field, "xdat:element_name").text = element
        ET.SubElement(field, "xdat:field_ID").text = field_id

    def create_criterias(parent, element, iterator):
        try:
            if isinstance(element, str):
                element = parent.set("method", element)
                create_criterias(parent, next(iterator), iterator)
            elif isinstance(element, tuple):
                if len(element) != 3:
                    raise errors.QueryError("The query-definition is incorrect.")
                else:
                    citeria = ET.SubElement(parent, "xdat:criteria", attrib={"override_value_formatting":"0"})
                    ET.SubElement(citeria, "xdat:schema_field").text=element[0]
                    ET.SubElement(citeria, "xdat:comparison_type").text=element[1]
                    ET.SubElement(citeria, "value").text=element[2]
                    create_criterias(parent, next(iterator), iterator)
            elif isinstance(element, list):
                child = ET.SubElement(parent, "xdat:child_set", method="AND")
                iter_child = iter(element)
                create_criterias(child, next(iter_child), iter_child)
        except StopIteration:
            pass

    iter_query = iter(query)
    where = ET.SubElement(root, "xdat:search_where", method="AND")
    create_criterias(where, next(iter_query), iter_query)

    return ET.ElementTree(root)