ded / express-limiter Goto Github PK
View Code? Open in Web Editor NEWRate limiting middleware for Express
License: MIT License
Rate limiting middleware for Express
License: MIT License
I tired to change the client to mongo and it failed. Has anyone used it?
Appreciate help. Thanks
Is there a way to send a custom response status and message without having to go inside the code?
Thanks for your advice.
Thanks to the express-limiter, I was able to implement an in memory solution..
If you want to implement in memory initially, the following hybrid codes could be chosen.
Any time you can move onto Redis without any change.
'use strict';
exports.limits = require('./limits');
'use strict';
// API Rate Limit
/**
* Limits in-memory data structure which stores all of the limits
*/
var limits = {}
exports.findAll = function () {
return limits
}
/**
* Returns a limit if it finds one, otherwise returns
* null if one is not found.
* @param key The key to the limit
* @param done The function to call next
* @returns The limit if found, otherwise returns null
*/
exports.get = function (key, done) {
var doc = limits[key]
var limit = doc ? JSON.stringify(doc.limit) : undefined
return done(null, limit)
}
/**
* Saves a limit using key, total, remainin and reset values.
* @param key The key value for the record that consists of client_id, path and method values (required)
* @param limit An object that contains total, remaining and reset fields (required)
* - total Allowed number of requests before getting rate limited
* - remaining Rest of allowed number of requests
* - reset The expiration date of the limit that is a javascript Date() object
* @param done Calls this with null always
* @returns returns this with null
*/
exports.set = function (key, limit, timeType, expire, done) {
limits[key] = { limit: JSON.parse(limit), timeType: timeType, expire: expire }
console.log(limits[key])
return done(null)
}
/**
* Deletes a limit
* @param key The limit to delete
* @param done returns this when done
*/
exports.delete = function (key, done) {
delete limits[key]
return done(null)
}
/**
* Removes expired limits. It does this by looping through them all
* and then removing the expired ones it finds.
* @param done returns this when done.
* @returns done
*/
exports.removeExpired = function (done) {
var limitsToDelete = []
var date = new Date()
for (var key in limits) {
if (limits.hasOwnProperty(key)) {
var doc = limits[key]
if (date > doc.expire) {
limitsToDelete.push(key)
}
}
}
for (var i = 0; i < limitsToDelete.length; ++i) {
console.log("Deleting limit:" + key)
delete limits[limitsToDelete[i]]
}
return done(null)
}
/**
* Removes all access limits.
* @param done returns this when done.
*/
exports.removeAll = function (done) {
limits = {}
return done(null)
}
/**
* Configuration of limits.
*
* total - Allowed number of requests before getting rate limited
* expiresIn - The time in seconds before the limit expires
* timeToCheckExpiredLimits - The time in seconds to check expired limits
*/
var min = 60000, // 1 minute in milliseconds
hour = 3600000; // 1 hour in milliseconds
exports.config = {
lookup: ['user.id'], // must be generated req.user object before. Or try 'connection.remoteAddress'
total: 150,
expire: 10 * min,
timeToRemoveExpiredLimits: 24 * hour
}
var config = require('./db').limits.config;
module.exports = function (app, db) {
return function (opts) {
var middleware = function (req, res, next) {
// If there is no opts object create ones
// and set the default properties
if(!opts) {
opts = { }
}
if(!opts.lookup) {
opts.lookup = config.lookup
}
if(!opts.total) {
opts.total = config.total
}
if(!opts.expire) {
opts.expire = config.expire
}
if (opts.whitelist && opts.whitelist(req)) return next()
opts.lookup = Array.isArray(opts.lookup) ? opts.lookup : [opts.lookup]
var lookups = opts.lookup.map(function (item) {
return item.split('.').reduce(function (prev, cur) {
return prev[cur]
}, req)
}).join(':')
var path = opts.path || req.path
var method = (opts.method || req.method).toLowerCase()
var key = path + ':' + method + ':' + lookups
db.get(key, function (err, limit) {
if (err && opts.ignoreErrors) return next()
var now = Date.now()
limit = limit ? JSON.parse(limit) : {
total: opts.total,
remaining: opts.total,
reset: now + opts.expire
}
if (now > limit.reset) {
limit.reset = now + opts.expire
limit.remaining = opts.total
}
// do not allow negative remaining
limit.remaining = Math.max(Number(limit.remaining) - 1, 0)
db.set(key, JSON.stringify(limit), 'PX', opts.expire, function (e) {
if (!opts.skipHeaders) {
res.set('X-RateLimit-Limit', limit.total)
res.set('X-RateLimit-Remaining', limit.remaining)
res.set('X-RateLimit-Reset', Math.ceil(limit.reset / 1000)) // UTC epoch seconds
}
if (limit.remaining) return next()
var after = (limit.reset - Date.now()) / 1000
if (!opts.skipHeaders) res.set('Retry-After', after)
res.status(429).send('Rate limit exceeded')
})
})
}
if (opts && opts.method && opts.path) app[opts.method](opts.path, middleware)
return middleware
}
}
Still this is express-limiter ! and you can use all examples on the main page.
I've implemented as a middleware.
var express = require('express'),
mongoose = require('mongoose'),
api = express.Router();
/* in memory express-limiter section */
var db = require('./db')
var limiter = require('express-limiter')(api, db.limits)
api.get("/test", limiter(), function(req, res) {
res.json([
{ value: 'foo' },
{ value: 'bar' },
{ value: 'baz' }
])
})
module.exports = api
This time you have to write as a middleware after any Authetication middleware. Because req.user
object is used to find lookup
parameter's value as user.id
.
var express = require('express')
var api = express()
/* in memory express-limiter section */
var db = require('./db')
var limiter = require('./rateLimiter')(api, db.limits)
// MIDDLEWARES
api.all('*', auth.isBearerAuthenticated)
api.use(limiter()) /* ta daa!.. */
I've added the following codes to my app.js
file
var limitConfig = db.limits.config
setInterval(function () {
db.limits.removeExpired(function (err) {
if (err) { console.error("Error removing expired limits") }
})}, limitConfig.timeToCheckExpiredLimits // once every 24 hours
)
The whitelist feature is great, however my function need to be able to return a Promise, something like the example below :/
whitelist: function(request) {
return new Promise(function(resolve, reject) {
if (request.context && request.context.user) {
api.user.permitsAuthorizationPolicy({user: request.context.user, policy: '*'}).then(resolve).catch(function(error) {
reject();
}
} else {
reject();
}
});
}
Right now, without this support, I have middleware which is checking the policy, and saving it to the request object, then checking that boolean value within the whitelist function. It's tedious imo
Will you allow this functionality please as I'm sure others would want it too!
The documentation states that the path and method are optional; however, when applying the express-limiter middleware globally, both of these options MUST be set. Otherwise, the middleware will not be applied to any routes (e.g., fail open).
The problem lies in the following lines of source code:
if (opts.method && opts.path) app[opts.method](opts.path, middleware)
return middleware
If the express-limiter method AND path options are both set then the middleware is applied to the corresponding Express route. Otherwise, the middleware is simply returned. If the express-limiter instance is being applied globally, then it will not be applied to any routes and will fail open.
This was probably done in an attempt to automatically detect whether the middleware is being applied globally vs directly into a route. If this is intended, then the documentation needs to be updated to warn that the path AND method MUST be set when applying the middleware globally.
I "consoled" the block about the lookup option being a function:
if (typeof(opts.lookup) === 'function') {
console.log('1:' + opts.lookup);
middleware = function (middleware, req, res, next) {
console.log('2:' + opts.lookup);
return opts.lookup(req, res, opts, function () {
console.log('3:' + opts.lookup);
return middleware(req, res, next)
})
}.bind(this, middleware)
}
Then I booted the server:
Tue Sep 29 2015 12:38:06 GMT-0400 (EDT): 1:function (req, res, opts, next) {
if (req.body && req.body.uuid) {
opts.lookup = ['body.uuid', 'params.uid'];
} else {
opts.lookup = ['connection.remoteAddress', 'params.uid'];
opts.total = 100;
}
return next();
}
Then I ran the first request:
2:function (req, res, opts, next) {
if (req.body && req.body.uuid) {
opts.lookup = ['body.uuid', 'params.uid'];
} else {
opts.lookup = ['connection.remoteAddress', 'params.uid'];
opts.total = 100;
}
return next();
}
3:connection.remoteAddress,params.uid
Then I ran the second request:
2:connection.remoteAddress,params.uid
Tue Sep 29 2015 12:38:23 GMT-0400 (EDT): TypeError: Property 'lookup' of object #<Object> is not a function
Notice the second go console log 2 has been updated to what was console log 3's.
Is there a way to limit all endpoints on my application except one? Or how can I set all endpoints to 5 requests per second, but a specific endpoint limit it at 8 requests per second?
It is possible to enable two stages of checking, for example the connection.remoteAddress. Can the user access up to 10x per minute AND at most 300x per day?
this example not works:
// with a function for dynamic-ness
limiter({
lookup: function(req, res, opts, next) {
if (validApiKey(req.query.api_key)) {
opts.lookup = 'query.api_key'
opts.total = 100
} else {
opts.lookup = 'connection.remoteAddress'
opts.total = 10
}
return next()
}
})
app.use('/api', limiter({
lookup: function(req, res, opts, next) {
opts.lookup = 'connection.remoteAddress';
opts.total = 1000;
return next();
},
}));
Got TypeError: opts.lookup is not a function
Also, the github repo is limiter
and the npm module is limitter
. I'd be happy to submit a PR for the docs, but cannot do much about the module/repo naming. ๐
Hello,
I recently found out that when using lookup
as a function, the param opts
is mutable.
That means, if you modify it, subsequent request made by same user or different users inherit the change you have made in it.
This is a very dangerous thing and as it is not documented, I suppose it was not made to be that way.
Here is simple way to reproduce:
lookup
in combinaison with total
total
computed for the previous request and not default one set in the root object.context('mutation', () => {
let express;
let app;
let limiter;
before(() => {
express = require('express');
app = express();
limiter = subject(app, redis);
limiter({
path: '*',
method: 'all',
lookup: function(req, res, opts, next) {
opts.lookup = 'query.api_key';
if (req.method === 'GET') {
opts.total = 20;
}
return next();
},
total: 3,
expire: 1000 * 60 * 60,
});
app
.get('/route', function(req, res) {
res.send(200, 'hello');
})
.post('/route', function(req, res) {
res.send(200, 'hello');
});
});
it('should have a special total', function(done) {
request(app)
.get('/route?api_key=foobar')
.expect('X-RateLimit-Limit', 20)
.expect('X-RateLimit-Remaining', 19)
.expect(200, function(e) {
done(e);
});
});
// -> This test will fail
// X-RateLimit-Limit will be equal to 20
it('should rollback to default', function(done) {
request(app)
.post('/route?api_key=foobar')
.expect('X-RateLimit-Limit', 3)
.expect('X-RateLimit-Remaining', 2)
.expect(200, function(e) {
done(e);
});
});
});
Has the package seems not maintained, I assume it's best to emphasis this issue ๐จ
Best regards
Can i implement two rules on same route? Thank you in advance
Line 37 in 99c10ed
You will always be limited to one less than the limit you set. For example, if I set limit to 1
it will decrement remaining
, save the JSON to Redis, and then once it hits the line above, remaining will be 0 and it sends a 429 response.
skipHeaders
defaults to false.
This means the entire world is getting these headers, which exposes internal implementation details and is thus a security flaw:
X-RateLimit-Limit: 20
X-RateLimit-Remaining: 19
X-RateLimit-Reset: 1510250052
X-Request-Id: da62f2a0-c576-11e7-b7fc-89bce46f8f85
Please consider changing the default in the next major release.
See this article about unnecessary exposure of implementation details.
I have an issue with the middleware feature when "expire" option is not specified. X-RateLimit-Remaning is only decremented by 1.
When I take a look to redis monitor with the command below
redis-cli monitor
I don't see any SET event, unlike the case when the "expire" option is specified.
Is it Redis bug function ?
The code :
var option = {
lookup : 'user.id',
total : '10'
}
router.get('/', limiter(option), action);
Hi,
I have express-limiter in place and it works as a charm .
The question is that for SEO reasons, I'd like to whitelist:
1 - crawlers that will index the site
2 - phantomjs that I use to prerender angular pages
Do you have any experience on that ?
I have the impression that I can work a solution for the phantomjs (as it's ran locally) relying in whitelist + connection.remoteAddress. But, I cannot find a solution for crawlers (other headers ? which ?)
Thanks a lot for the help and congrats for the module, it's really great!
Best regards!
I have a project where this module would be pretty much perfect, but I'm using Memcache instead of Redis (via https://github.com/3rd-Eden/memcached). As far as I can tell, it should be trivial to adapt since the Memcache module also has get()
and set()
functions. I'd love to submit a pull request if it's something you have any interest in adding, but...
I'm confused by the db.set()
call. Specifically the 3rd parameter: 'PX'
. What is that? I haven't used the redis module, but looking at the docs, I don't see where that parameter is necessary or even allowed. I'm sure I'm missing something, but that param keeps the module from working with memcache.
Can you help me understand what's happening there so I know how to proceed (again, assuming memcache compatibility is of interest to you).
Thanks.
I set the rate limit at 3 TPS. When I send 4 TPS, it works and I see 3TPS rate honored. However, when I put this into production, as the volumes go up, 3 TPS begins to break.
AT 10 TPS INPUT, I see ~4 TPS getting through the rate limiter.
AT 40 TPS INPUT, I see ~6 TPS getting through the rate limiter.
Any help is appreciated.
var limiter = require('express-limiter')(router, client);
limiter({
path: '/',
method: 'post',
onRateLimited: function (req, res, next) {
next({ message: 'Rate limit exceeded', status: 429 })
},
lookup: function(req, res, opts, next) {
opts.lookup = 'headers.id';
opts.total = 3; // 1 TPS
}
return next();
},
expire: 1000 // (1 seconds)
});
How can I use the mongodb with the express-limitter using native mongodb driver
user A -> rate limit A, user B -> rate limit B.
How do i achieve this with express-limiter
?
Ignore. My Redis service as fading in and out of service.
Hi - Thank you for taking the time to create this!
How do I protect multiple endpoints in express?
const rateLimiter = RateLimiter(app, app.locals.redisClient0);
rateLimiter({
path: '/api/fetch',
method: 'post',
lookup: 'headers.x-forwarded-for',
total: 120,
expire: 1000 * 60 * 60
});
It seems like the path is a string, not an array. Please let me know - thanks!
The lookup
right now is set to an array of values. This is useful, but is always AND not OR. What if I wanted to rate limit by an api key if available and valid, or fallback to an ip? This could potentially use a function instead of an array, that took in the request.
What if falling back to an ip would change how many requests they could make in total
? I suppose the function above could also pass in the config of the express limiter.
I'd be happy to make a pr once these concepts have been approved.
A declarative, efficient, and flexible JavaScript library for building user interfaces.
๐ Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. ๐๐๐
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google โค๏ธ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.