queries.py 7.81 KB
Newer Older
Franziska Koehn's avatar
Franziska Koehn committed
1
2
3
4
5
"""
:Author: Franziska Koehn
:Created: 2015/01/13

This module includes functions around sending and defining queries.
Franziska Koehn's avatar
Franziska Koehn committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

to query the xnat server define:

one string including one Root-Type like:

.. sourcecode:: python

   'xnat:mrScanData'

one list including the search-constraints like:

.. code-block:: python

    [
        ('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
        ('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
        'AND',
        [('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
         ('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
         'OR'
         ]
    ]

a list of search-fields like:

.. code-block:: python

    [ 'xnat:mrScanData/TYPE','xnat:mrScanData/ID']

Franziska Koehn's avatar
Franziska Koehn committed
35
36
"""

Franziska Koehn's avatar
Franziska Koehn committed
37
38
39
from threading import Lock
requests_lock = Lock()

40
def get_query_methods():
Franziska Koehn's avatar
Franziska Koehn committed
41
    """Returns all applicable methods for creating a query."""
42
43
44
    return ["AND", "OR"]

def get_operators():
Franziska Koehn's avatar
Franziska Koehn committed
45
    """Returns all applicable Operators for creating a query."""
46
47
    return ["LIKE", ">", "<", "<=", ">=", "="]

Franziska Koehn's avatar
Franziska Koehn committed
48

49
def download_async(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
50
    """
Franziska Koehn's avatar
Franziska Koehn committed
51
    Downloads file by using threads (So the GUI will not be blocked while downloading).
Franziska Koehn's avatar
Franziska Koehn committed
52
53

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
54
55
56
57
58
59
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
60
61
        :cb_args: args for function cb
    """
Franziska Koehn's avatar
Franziska Koehn committed
62
    from threading import Thread
63
    download_thread = Thread(target=download, args=(result, host, creds, rest, dest_folder, cb, cb_args))
Franziska Koehn's avatar
Franziska Koehn committed
64
65
    download_thread.start()
    return download_thread
Franziska Koehn's avatar
Franziska Koehn committed
66

67
def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
68
    """
Franziska Koehn's avatar
Franziska Koehn committed
69
    Downloads a Result.
Franziska Koehn's avatar
Franziska Koehn committed
70
71

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
72
73
74
75
76
77
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
78
79
80
        :cb_args: args for function cb
    """

Franziska Koehn's avatar
Franziska Koehn committed
81
    requests_lock.acquire()
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

    import re
    import os

    names=[]
    def subfunc(f):
        key = f.group(0).strip("{}")
        r = result[key]
        names.append(r)
        return r
    ret = re.sub('\{\w+\}', subfunc, rest)

    path = os.path.join(dest_folder, '-'.join(names))
    url = "%s%s" % (host, ret)

    download_file(url, creds, path)
Franziska Koehn's avatar
Franziska Koehn committed
98
99
    requests_lock.release()
    cb(*cb_args)
100

101
def download_file(url, creds, path):
Franziska Koehn's avatar
Franziska Koehn committed
102
    """
Franziska Koehn's avatar
Franziska Koehn committed
103
104
105
106
    Downloads a file from given 'url' and saves it to 'path'.

    Returns True if download was finished without problems
    Returns False if download raised an exception.
Franziska Koehn's avatar
Franziska Koehn committed
107
108

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
109
110
111
        :url: str, host/REST-API with filled values
        :creds: tuple(user-name, user-password), credentials
        :path: str, folder where the downloaded files will be saved
Franziska Koehn's avatar
Franziska Koehn committed
112
113
    """

Franziska Koehn's avatar
Franziska Koehn committed
114
115
116
117
    import requests
    from base64 import b64encode
    from requests.auth import HTTPBasicAuth

118
119
    if not path.endswith(".zip"):
        path += ".zip"
Franziska Koehn's avatar
Franziska Koehn committed
120

121
    user, passw = creds
Franziska Koehn's avatar
Franziska Koehn committed
122
123

    try:
124
        with open(path, 'wb') as handle:
Franziska Koehn's avatar
Franziska Koehn committed
125
126
127
128
129
130
131
            response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
            if not response.ok:
                raise ValueError(response.status_code)
            for block in response.iter_content(1024):
                if not block:
                    break
                handle.write(block)
Franziska Koehn's avatar
Franziska Koehn committed
132
        return True
Franziska Koehn's avatar
Franziska Koehn committed
133
    except IOError as e:
134
        print "Error writing file %s, %s" % (path, e)
Franziska Koehn's avatar
Franziska Koehn committed
135
136
    except ValueError as e:
        print "Error downloading file %s, Status Code %s" % (url, e)
Franziska Koehn's avatar
Franziska Koehn committed
137
    return False
Franziska Koehn's avatar
Franziska Koehn committed
138
139


140
141
142
143
144
145
146
147
def get_xnat_server_connection(force=False, host="", user="", passw=""):
    from pyxnat import Interface
    from tempfile import mkdtemp
    import xsa.errors

    if get_xnat_server_connection.cache and not force:
        return get_xnat_server_connection.cache
    elif force and host=="":
148
        raise xsa.errors.ServerNotFoundError("The host-address was not defined.")
149
150
151
152
153
154
155
    elif force and (user=="" or passw==""):
        raise xsa.errors.UnauthorizedError("The user-name and/or the user-password were not defined.")

    disconnect_xnat_server()
    try:
        get_xnat_server_connection.cache = Interface(server=host, user=user, password=passw, cachedir=mkdtemp())
    except IndexError as e:
156
        raise xsa.errors.ServerNotFoundError("Can't reach host! Check your host-address and your internet/network connection.")
157
158
159
160
161
162
163
164
165
166
    return get_xnat_server_connection()
get_xnat_server_connection.cache = None

def disconnect_xnat_server():
    import xsa.errors
    try:
        get_xnat_server_connection.cache.disconnect()
        get_xnat_server_connection.cache = None
    except:
       return
167

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
def retry_qry(fun):
    def retry(*args, **kwargs):
        last_exception = None
        retries = 1
        while retries >= 0:
            try:
                return fun(*args, **kwargs)
            except Exception as e:
                print(("wrapped", e))
                get_xnat_server_connection.cache = None
                if type(last_exception) is type(e):
                    raise e
                last_exception = e
            finally:
                retries -= 1
    return retry

@retry_qry
186
def query_for_additional_data(rest, result, host, creds):
187
188
189
190

    import re
    def subfunc(f):
        key = f.group(0).strip("{}")
Franziska Koehn's avatar
Franziska Koehn committed
191
        return result[key]
192
193
    ret = re.sub('\{\w+\}', subfunc, rest)

194
    user, passw = creds
195
196
197
198
199
200
201
202
203
204
205
    central = get_xnat_server_connection(host=host, user=user, passw=passw)
    exp = central.select(ret)

    new_fields = exp.xpath("//xnat:field")
    new_result = result.copy()
    for f in new_fields:
        fieldname = f.get("name")
        value = exp.xpath("//xnat:field[@name='"+fieldname+"']/child::text()")
        new_result[fieldname] = ''.join(value).replace("\n","")
    return new_result

206
@retry_qry
207
def search_for(host, root_element, constraints, search_fields, user, passw):
Franziska Koehn's avatar
Franziska Koehn committed
208
209
210
211
    """
    Does a search for given values. raises xsa -Exceptions

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
212
213
214
215
216
217
218
219
220
221
222
223
224
        :host: str, host-address
        :root_element: str, Root-Type of search
        :constraints: list, constraints of query
        :search_fields: list of str, fields which will be returned from server
        :user: str, user-name
        :passw: str, user-password

    **Raises**
        :xsa.errors.ServerNotFoundError:
        :xsa.errors.UnauthorizedError:
        :xsa.errors.ResponseNotReady:
        :xsa.errors.QueryError:

225

Franziska Koehn's avatar
Franziska Koehn committed
226
    """
Franziska Koehn's avatar
Franziska Koehn committed
227
    import xsa.errors
228

229
230
    from pyxnat.core import errors
    from httplib2 import ServerNotFoundError
231
232
233
    from httplib import ResponseNotReady

    from tempfile import mkdtemp
234
    tmp_dir=mkdtemp()
235

236
    central = get_xnat_server_connection(host=host, user=user, passw=passw)
237

238
239
240
241
    result = []

    try:
        result =  central.select(root_element,search_fields).where(constraints)
242
243
    except errors.DatabaseError as e:
        if '401' in str(e):
244
            raise xsa.errors.UnauthorizedError("Unauthorizied attempt. Check your user-name and -password")
Franziska Koehn's avatar
Franziska Koehn committed
245
246
        else:
            raise e
247
    except ServerNotFoundError:
248
        raise xsa.errors.ServerNotFoundError("Can't reach host! Check your host-address and your internet/network connection.")
249
    except ResponseNotReady:
250
        raise xsa.errors.ResponseNotReady("Please check your host-address")
251
252

    if result == []:
Franziska Koehn's avatar
Franziska Koehn committed
253
        raise xsa.errors.QueryError("Please check your query.")
254
255
256
257
258
259

    try:
        from shutil import rmtree
        rmtree(tmp_dir)
    except:
        pass
260

261
    return result