queries.py 7.19 KB
Newer Older
Franziska Koehn's avatar
Franziska Koehn committed
1
2
3
4
5
"""
:Author: Franziska Koehn
:Created: 2015/01/13

This module includes functions around sending and defining queries.
Franziska Koehn's avatar
Franziska Koehn committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

to query the xnat server define:

one string including one Root-Type like:

.. sourcecode:: python

   'xnat:mrScanData'

one list including the search-constraints like:

.. code-block:: python

    [
        ('xnat:mrScanData/TYPE', 'LIKE', '%t2%'),
        ('xnat:mrScanData/PARAMETERS_FLIP', '>=', '10'),
        'AND',
        [('xnat:mrScanData/PARAMETERS_TE', '>', '2.0'),
         ('xnat:mrScanData/PARAMETERS_TE', '<', '2.0'),
         'OR'
         ]
    ]

a list of search-fields like:

.. code-block:: python

    [ 'xnat:mrScanData/TYPE','xnat:mrScanData/ID']

Franziska Koehn's avatar
Franziska Koehn committed
35
36
"""

Franziska Koehn's avatar
Franziska Koehn committed
37
38
39
from threading import Lock
requests_lock = Lock()

40
def get_query_methods():
Franziska Koehn's avatar
Franziska Koehn committed
41
    """Returns all applicable methods for creating a query."""
42
43
44
    return ["AND", "OR"]

def get_operators():
Franziska Koehn's avatar
Franziska Koehn committed
45
    """Returns all applicable Operators for creating a query."""
46
47
    return ["LIKE", ">", "<", "<=", ">=", "="]

Franziska Koehn's avatar
Franziska Koehn committed
48

49
def download_async(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
50
    """
Franziska Koehn's avatar
Franziska Koehn committed
51
    Downloads file by using threads (So the GUI will not be blocked while downloading).
Franziska Koehn's avatar
Franziska Koehn committed
52
53

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
54
55
56
57
58
59
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
60
61
        :cb_args: args for function cb
    """
Franziska Koehn's avatar
Franziska Koehn committed
62
    from threading import Thread
63
    download_thread = Thread(target=download, args=(result, host, creds, rest, dest_folder, cb, cb_args))
Franziska Koehn's avatar
Franziska Koehn committed
64
65
    download_thread.start()
    return download_thread
Franziska Koehn's avatar
Franziska Koehn committed
66

67
def download(result, host, creds, rest, dest_folder='', cb=(lambda *_: None), cb_args=()):
Franziska Koehn's avatar
Franziska Koehn committed
68
    """
Franziska Koehn's avatar
Franziska Koehn committed
69
    Downloads a Result.
Franziska Koehn's avatar
Franziska Koehn committed
70
71

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
72
73
74
75
76
77
        :result: dict, resultset from which will be downloaded
        :host: str, address of host
        :creds: tuple (username, userpassword), credentials (including user-name and -password)
        :rest: str, REST-API definition
        :dest_folder: str, folder where the downloaded files will be saved
        :cb: function, that will be called, when download is finished with the arguments from cb_args
Franziska Koehn's avatar
Franziska Koehn committed
78
79
80
        :cb_args: args for function cb
    """

Franziska Koehn's avatar
Franziska Koehn committed
81
    requests_lock.acquire()
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

    import re
    import os

    names=[]
    def subfunc(f):
        key = f.group(0).strip("{}")
        r = result[key]
        names.append(r)
        return r
    ret = re.sub('\{\w+\}', subfunc, rest)

    path = os.path.join(dest_folder, '-'.join(names))
    url = "%s%s" % (host, ret)

    download_file(url, creds, path)
Franziska Koehn's avatar
Franziska Koehn committed
98
99
    requests_lock.release()
    cb(*cb_args)
100

101
def download_file(url, creds, path):
Franziska Koehn's avatar
Franziska Koehn committed
102
    """
Franziska Koehn's avatar
Franziska Koehn committed
103
104
105
106
    Downloads a file from given 'url' and saves it to 'path'.

    Returns True if download was finished without problems
    Returns False if download raised an exception.
Franziska Koehn's avatar
Franziska Koehn committed
107
108

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
109
110
111
        :url: str, host/REST-API with filled values
        :creds: tuple(user-name, user-password), credentials
        :path: str, folder where the downloaded files will be saved
Franziska Koehn's avatar
Franziska Koehn committed
112
113
    """

Franziska Koehn's avatar
Franziska Koehn committed
114
115
116
117
    import requests
    from base64 import b64encode
    from requests.auth import HTTPBasicAuth

118
119
    if not path.endswith(".zip"):
        path += ".zip"
Franziska Koehn's avatar
Franziska Koehn committed
120

121
    user, passw = creds
Franziska Koehn's avatar
Franziska Koehn committed
122
123

    try:
124
        with open(path, 'wb') as handle:
Franziska Koehn's avatar
Franziska Koehn committed
125
126
127
128
129
130
131
            response = requests.get(url, stream=True, auth=HTTPBasicAuth(user, passw))
            if not response.ok:
                raise ValueError(response.status_code)
            for block in response.iter_content(1024):
                if not block:
                    break
                handle.write(block)
Franziska Koehn's avatar
Franziska Koehn committed
132
        return True
Franziska Koehn's avatar
Franziska Koehn committed
133
    except IOError as e:
134
        print "Error writing file %s, %s" % (path, e)
Franziska Koehn's avatar
Franziska Koehn committed
135
136
    except ValueError as e:
        print "Error downloading file %s, Status Code %s" % (url, e)
Franziska Koehn's avatar
Franziska Koehn committed
137
    return False
Franziska Koehn's avatar
Franziska Koehn committed
138
139


140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def get_xnat_server_connection(force=False, host="", user="", passw=""):
    from pyxnat import Interface
    from tempfile import mkdtemp
    import xsa.errors

    if get_xnat_server_connection.cache and not force:
        return get_xnat_server_connection.cache
    elif force and host=="":
        raise xsa.errors.ServerNotFoundError("The host was not defined.")
    elif force and (user=="" or passw==""):
        raise xsa.errors.UnauthorizedError("The user-name and/or the user-password were not defined.")

    disconnect_xnat_server()
    try:
        get_xnat_server_connection.cache = Interface(server=host, user=user, password=passw, cachedir=mkdtemp())
    except IndexError as e:
        raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
    return get_xnat_server_connection()
get_xnat_server_connection.cache = None

def disconnect_xnat_server():
    import xsa.errors
    try:
        get_xnat_server_connection.cache.disconnect()
        get_xnat_server_connection.cache = None
    except:
       return
167
168
169
170
171
172

def query_for_additional_data(rest, result, host, user, passw):

    import re
    def subfunc(f):
        key = f.group(0).strip("{}")
Franziska Koehn's avatar
Franziska Koehn committed
173
        return result[key]
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    ret = re.sub('\{\w+\}', subfunc, rest)

    central = get_xnat_server_connection(host=host, user=user, passw=passw)
    exp = central.select(ret)

    new_fields = exp.xpath("//xnat:field")
    new_result = result.copy()
    for f in new_fields:
        fieldname = f.get("name")
        value = exp.xpath("//xnat:field[@name='"+fieldname+"']/child::text()")
        new_result[fieldname] = ''.join(value).replace("\n","")
    return new_result


188
def search_for(host, root_element, constraints, search_fields, user, passw):
Franziska Koehn's avatar
Franziska Koehn committed
189
190
191
192
    """
    Does a search for given values. raises xsa -Exceptions

    **Parameters**
Franziska Koehn's avatar
Franziska Koehn committed
193
194
195
196
197
198
199
200
201
202
203
204
205
        :host: str, host-address
        :root_element: str, Root-Type of search
        :constraints: list, constraints of query
        :search_fields: list of str, fields which will be returned from server
        :user: str, user-name
        :passw: str, user-password

    **Raises**
        :xsa.errors.ServerNotFoundError:
        :xsa.errors.UnauthorizedError:
        :xsa.errors.ResponseNotReady:
        :xsa.errors.QueryError:

206

Franziska Koehn's avatar
Franziska Koehn committed
207
    """
Franziska Koehn's avatar
Franziska Koehn committed
208
    import xsa.errors
209

210
211
    from pyxnat.core import errors
    from httplib2 import ServerNotFoundError
212
213
214
    from httplib import ResponseNotReady

    from tempfile import mkdtemp
215
    tmp_dir=mkdtemp()
216

217
    central = get_xnat_server_connection(host=host, user=user, passw=passw)
218

219
220
221
222
    result = []

    try:
        result =  central.select(root_element,search_fields).where(constraints)
223
224
    except errors.DatabaseError as e:
        if '401' in str(e):
Franziska Koehn's avatar
Franziska Koehn committed
225
226
227
            raise xsa.errors.UnauthorizedError("Unauthorizied attempt. Check your User and Password")
        else:
            raise e
228
    except ServerNotFoundError:
Franziska Koehn's avatar
Franziska Koehn committed
229
        raise xsa.errors.ServerNotFoundError("Server not found, check your host-address.")
230
    except ResponseNotReady:
Franziska Koehn's avatar
Franziska Koehn committed
231
        raise xsa.errors.ResponseNotReady("Please check your Host-Address")
232
233

    if result == []:
Franziska Koehn's avatar
Franziska Koehn committed
234
        raise xsa.errors.QueryError("Please check your query.")
235
236
237
238
239
240

    try:
        from shutil import rmtree
        rmtree(tmp_dir)
    except:
        pass
241

242
    return result