Pārlūkot izejas kodu

Merge pull request #800 from dalf/searchpy5

search.py : Fix and some minor modifications
Adam Tauber 8 gadus atpakaļ
vecāks
revīzija
3d8c9bab96
2 mainītis faili ar 14 papildinājumiem un 17 dzēšanām
  1. 1
    0
      searx/results.py
  2. 13
    17
      searx/search.py

+ 1
- 0
searx/results.py Parādīt failu

@@ -133,6 +133,7 @@ class ResultContainer(object):
133 133
 
134 134
     def extend(self, engine_name, results):
135 135
         for result in list(results):
136
+            result['engine'] = engine_name
136 137
             if 'suggestion' in result:
137 138
                 self.suggestions.add(result['suggestion'])
138 139
                 results.remove(result)

+ 13
- 17
searx/search.py Parādīt failu

@@ -37,7 +37,7 @@ logger = logger.getChild('search')
37 37
 number_of_searches = 0
38 38
 
39 39
 
40
-def send_http_request(engine, request_params, timeout_limit):
40
+def send_http_request(engine, request_params, start_time, timeout_limit):
41 41
     # for page_load_time stats
42 42
     time_before_request = time()
43 43
 
@@ -62,9 +62,10 @@ def send_http_request(engine, request_params, timeout_limit):
62 62
 
63 63
     # is there a timeout (no parsing in this case)
64 64
     timeout_overhead = 0.2  # seconds
65
-    search_duration = time() - request_params['started']
65
+    time_after_request = time()
66
+    search_duration = time_after_request - start_time
66 67
     if search_duration > timeout_limit + timeout_overhead:
67
-        raise Timeout(response=response)
68
+        raise requests.exceptions.Timeout(response=response)
68 69
 
69 70
     with threading.RLock():
70 71
         # no error : reset the suspend variables
@@ -72,14 +73,14 @@ def send_http_request(engine, request_params, timeout_limit):
72 73
         engine.suspend_end_time = 0
73 74
         # update stats with current page-load-time
74 75
         # only the HTTP request
75
-        engine.stats['page_load_time'] += time() - time_before_request
76
+        engine.stats['page_load_time'] += time_after_request - time_before_request
76 77
         engine.stats['page_load_count'] += 1
77 78
 
78 79
     # everything is ok : return the response
79 80
     return response
80 81
 
81 82
 
82
-def search_one_request(engine, query, request_params, timeout_limit):
83
+def search_one_request(engine, query, request_params, start_time, timeout_limit):
83 84
     # update request parameters dependent on
84 85
     # search-engine (contained in engines folder)
85 86
     engine.request(query, request_params)
@@ -92,24 +93,21 @@ def search_one_request(engine, query, request_params, timeout_limit):
92 93
         return []
93 94
 
94 95
     # send request
95
-    response = send_http_request(engine, request_params, timeout_limit)
96
+    response = send_http_request(engine, request_params, start_time, timeout_limit)
96 97
 
97 98
     # parse the response
98 99
     response.search_params = request_params
99 100
     return engine.response(response)
100 101
 
101 102
 
102
-def search_one_request_safe(engine_name, query, request_params, result_container, timeout_limit):
103
-    start_time = time()
103
+def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
104 104
     engine = engines[engine_name]
105 105
 
106 106
     try:
107 107
         # send requests and parse the results
108
-        search_results = search_one_request(engine, query, request_params, timeout_limit)
108
+        search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
109 109
 
110 110
         # add results
111
-        for result in search_results:
112
-            result['engine'] = engine_name
113 111
         result_container.extend(engine_name, search_results)
114 112
 
115 113
         # update engine time when there is no exception
@@ -131,7 +129,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container
131 129
                          "(search duration : {1} s, timeout: {2} s) : {3}"
132 130
                          .format(engine_name, search_duration, timeout_limit, e.__class__.__name__))
133 131
             requests_exception = True
134
-        if (issubclass(e.__class__, requests.exceptions.RequestException)):
132
+        elif (issubclass(e.__class__, requests.exceptions.RequestException)):
135 133
             # other requests exception
136 134
             logger.exception("engine {0} : requests exception"
137 135
                              "(search duration : {1} s, timeout: {2} s) : {3}"
@@ -151,14 +149,13 @@ def search_one_request_safe(engine_name, query, request_params, result_container
151 149
         return False
152 150
 
153 151
 
154
-def search_multiple_requests(requests, result_container, timeout_limit):
155
-    start_time = time()
152
+def search_multiple_requests(requests, result_container, start_time, timeout_limit):
156 153
     search_id = uuid4().__str__()
157 154
 
158 155
     for engine_name, query, request_params in requests:
159 156
         th = threading.Thread(
160 157
             target=search_one_request_safe,
161
-            args=(engine_name, query, request_params, result_container, timeout_limit),
158
+            args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
162 159
             name=search_id,
163 160
         )
164 161
         th._engine_name = engine_name
@@ -368,7 +365,6 @@ class Search(object):
368 365
             request_params = default_request_params()
369 366
             request_params['headers']['User-Agent'] = user_agent
370 367
             request_params['category'] = selected_engine['category']
371
-            request_params['started'] = start_time
372 368
             request_params['pageno'] = search_query.pageno
373 369
 
374 370
             if hasattr(engine, 'language') and engine.language:
@@ -388,7 +384,7 @@ class Search(object):
388 384
 
389 385
         if requests:
390 386
             # send all search-request
391
-            search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
387
+            search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
392 388
             start_new_thread(gc.collect, tuple())
393 389
 
394 390
         # return results, suggestions, answers and infoboxes