This is an automated email from the ASF dual-hosted git repository. csantanapr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-apigateway.git
The following commit(s) were added to refs/heads/master by this push: new 47c3bbe data layer (#198) 47c3bbe is described below commit 47c3bbeb496675857c9a165a1d56940c123a032e Author: Taylor King <taylorbradleyk...@gmail.com> AuthorDate: Thu May 25 15:03:40 2017 -0400 data layer (#198) --- api-gateway.conf | 1 + conf.d/managed_endpoints.conf | 3 +- conf.d/management_apis.conf | 20 ++- scripts/lua/lib/dataStore.lua | 141 ++++++++++++++++++ scripts/lua/lib/redis.lua | 190 ++++++++++++++---------- scripts/lua/management/lib/apis.lua | 42 +++--- scripts/lua/management/lib/resources.lua | 19 ++- scripts/lua/management/lib/tenants.lua | 32 ++-- scripts/lua/management/lib/validation.lua | 13 +- scripts/lua/management/routes/apis.lua | 79 +++++----- scripts/lua/management/routes/subscriptions.lua | 4 +- scripts/lua/management/routes/tenants.lua | 56 ++++--- scripts/lua/oauth/facebook.lua | 30 ++-- scripts/lua/oauth/github.lua | 20 +-- scripts/lua/oauth/google.lua | 28 ++-- scripts/lua/oauth/mock.lua | 5 +- scripts/lua/policies/rateLimit.lua | 7 +- scripts/lua/policies/security.lua | 4 +- scripts/lua/policies/security/apiKey.lua | 32 ++-- scripts/lua/policies/security/clientSecret.lua | 57 ++++--- scripts/lua/policies/security/oauth2.lua | 28 ++-- scripts/lua/routing.lua | 25 ++-- tests/scripts/lua/lib/redis.lua | 53 ++++--- tests/scripts/lua/routing.lua | 8 +- tests/scripts/lua/security.lua | 28 +++- 25 files changed, 556 insertions(+), 369 deletions(-) diff --git a/api-gateway.conf b/api-gateway.conf index bcdfcb0..e206e48 100644 --- a/api-gateway.conf +++ b/api-gateway.conf @@ -28,6 +28,7 @@ worker_rlimit_nofile 524288; # 100000 env REDIS_HOST; env REDIS_PORT; env REDIS_PASS; +env REDIS_TIMEOUT; env PUBLIC_MANAGEDURL_HOST; env PUBLIC_MANAGEDURL_PORT; env HOST; diff --git a/conf.d/managed_endpoints.conf b/conf.d/managed_endpoints.conf index 3d375e9..ba9c94e 100644 --- a/conf.d/managed_endpoints.conf +++ b/conf.d/managed_endpoints.conf @@ -79,7 +79,8 @@ server { access_by_lua_block { local routing = require "routing" - routing.processCall() + local ds = require "lib/dataStore" + routing.processCall(ds.init()) } proxy_pass $upstream; diff --git a/conf.d/management_apis.conf b/conf.d/management_apis.conf index 94e0f57..abcc241 100644 --- a/conf.d/management_apis.conf +++ b/conf.d/management_apis.conf @@ -35,36 +35,46 @@ server { location ~ /(?<version>v2)/(?<tenantId>[^/]*)/apis/?(?<api_id>[^/]*)?/?(?<query>[^/]*)? { access_by_lua_block { + local ds = require('lib/dataStore') + local dataStore = ds.init() local apis = require("management/routes/apis") - apis.requestHandler() + apis.requestHandler(dataStore) } } location ~ /(?<version>v1)/apis/?(?<api_id>[^/]*)?/?(?<query>[^/]*)? { access_by_lua_block { local apis = require("management/routes/apis") - apis.requestHandler() + local ds = require('lib/dataStore') + local dataStore = ds.init() + apis.requestHandler(dataStore) } } location ~ /v1/tenants/?(?<tenant_id>[^/]*)?/?(?<query>[^/]*)? { access_by_lua_block { local tenants = require("management/routes/tenants") - tenants.requestHandler() + local ds = require('lib/dataStore') + local dataStore = ds.init() + tenants.requestHandler(dataStore) } } location ~ /(?<version>v2)/(?<tenant_id>[^/]*)/subscriptions/?(?<client_id>[^/]*)?/? { access_by_lua_block { local subscriptions = require("management/routes/subscriptions") - subscriptions.requestHandler() + local ds = require('lib/dataStore') + local dataStore = ds.init() + subscriptions.requestHandler(dataStore) } } location ~ /(?<version>v1)/subscriptions/? { access_by_lua_block { local subscriptions = require("management/routes/subscriptions") - subscriptions.requestHandler() + local ds = require('lib/dataStore') + local dataStore = ds.init() + subscriptions.requestHandler(dataStore) } } diff --git a/scripts/lua/lib/dataStore.lua b/scripts/lua/lib/dataStore.lua new file mode 100644 index 0000000..52aa773 --- /dev/null +++ b/scripts/lua/lib/dataStore.lua @@ -0,0 +1,141 @@ + + + + + +local DATASTORE = os.getenv( 'DATASTORE') +local utils = require('lib/utils') + +if DATASTORE == nil then + DATASTORE = 'redis' +end +local _M = {} + +local DataStore = {} +function DataStore:init() + local o = {} + setmetatable(o, self) + self.__index = self + o.impl = require(utils.concatStrings({'lib/', DATASTORE})) + o.ds = o.impl.init() + return o +end + +-- right now just using this for the tests +function DataStore:initWithDriver(ds) +local o = {} + setmetatable(o, self) + self.__index = self + o.impl = require('lib/redis') + o.ds = ds + return o +end + +function DataStore:close() + return self.impl.close(self.ds) +end + +function DataStore:addAPI(id, apiObj, existingAPI) + return self.impl.addAPI(self.ds, id, apiObj, existingAPI) +end + +function DataStore:getAllAPIs() + return self.impl.getAllAPIs(self.ds) +end + +function DataStore:getAPI(id) + return self.impl.getAPI(self.ds, id) +end + +function DataStore:deleteAPI(id) + return self.impl.deleteAPI(self.ds, id) +end + +function DataStore:resourceToApi(resource) + return self.impl.resourceToApi(self.ds, resource) +end + +function DataStore:generateResourceObj(ops, apiId, tenantObj, cors) + return self.impl.generateResourceObj(ops, apiId, tenantObj, cors) +end + +function DataStore:createResource(key, field, resourceObj) + return self.impl.createResource(self.ds, key, field, resourceObj) +end + +function DataStore:addResourceToIndex(index, resourceKey) + return self.impl.addResourceToIndex(self.ds, index, resourceKey) +end + +function DataStore:deleteResourceFromIndex(index, resourceKey) + return self.impl.deleteResourceFromIndex(self.ds, index, resourceKey) +end +function DataStore:getResource(key, field) + return self.impl.getResource(self.ds, key, field) +end +function DataStore:getAllResources(tenantId) + return self.impl.getAllResources(self.ds, tenantId) +end +function DataStore:deleteResource(key, field) + return self.impl.deleteResource(self.ds, key, field) +end +function DataStore:addTenant(id, tenantObj) + return self.impl.addTenant(self.ds, id, tenantObj) +end +function DataStore:getAllTenants() + return self.impl.getAllTenants(self.ds) +end +function DataStore:getTenant(id) + return self.impl.getTenant(self.ds, id) +end +function DataStore:deleteTenant(id) + return self.impl.deleteTenant(self.ds, id) +end +function DataStore:createSubscription(key) + return self.impl.createSubscription(self.ds, key) +end +function DataStore:deleteSubscription(key) + return self.impl.deleteSubscription(self.ds, key) +end +function DataStore:healthCheck() + return self.impl.healthCheck(self.ds) +end +function DataStore:addSwagger(id, swagger) + return self.impl.addSwagger(self.ds, id, swagger) +end +function DataStore:getSwagger(id) + return self.impl.getSwagger(self.ds, id) +end + +function DataStore:deleteSwagger(id) + return self.impl.deleteSwagger(self.ds, id) +end + +function DataStore:getOAuthToken(provider, token) + return self.impl.getOAuthToken(self.ds, provider, token) +end + +function DataStore:saveOAuthToken(provider, token, body, ttl) + return self.impl.saveOAuthToken(self.ds, provider, token, body, ttl) +end + +function DataStore:exists(key) + return self.impl.exists(self.ds, key) +end + +function DataStore:setRateLimit(key, value, interval, expires) + return self.impl.setRateLimit(self.ds, key, value, interval, expires) +end +function DataStore:getRateLimit(key) + return self.impl.getRateLimit(self.ds, key) +end +-- to be removed in the future + + +function _M.init() + return DataStore:init() +end +function _M.initWithDriver(ds) + return DataStore:initWithDriver(ds) +end +return _M diff --git a/scripts/lua/lib/redis.lua b/scripts/lua/lib/redis.lua index 64fad46..c0b4d24 100644 --- a/scripts/lua/lib/redis.lua +++ b/scripts/lua/lib/redis.lua @@ -25,23 +25,33 @@ local cjson = require "cjson" local utils = require "lib/utils" local logger = require "lib/logger" local request = require "lib/request" -local lrucache +local lrucache local CACHE_SIZE local CACHE_TTL local c, err + +local REDIS_HOST = os.getenv("REDIS_HOST") +local REDIS_PORT = os.getenv("REDIS_PORT") +local REDIS_PASS = os.getenv("REDIS_PASS") +local REDIS_TIMEOUT = os.getenv("REDIS_TIMEOUT") +if REDIS_TIMEOUT == nil then + REDIS_TIMEOUT = 10000 +else + REDIS_TIMEOUT = tonumber(REDIS_TIMEOUT) +end local CACHING_ENABLED = os.getenv('CACHING_ENABLED') if CACHING_ENABLED then lrucache = require "resty.lrucache" CACHE_SIZE = tonumber(os.getenv('CACHE_SIZE')) CACHE_TTL = tonumber(os.getenv('CACHE_TTL')) c, err = lrucache.new(CACHE_SIZE) - if not c then + if not c then return error("Failed to initialize LRU cache" .. (err or "unknown")) - end -end + end +end -local REDIS_RETRY_COUNT = os.getenv('REDIS_RETRY_COUNT') or 4 +local REDIS_RETRY_COUNT = os.getenv('REDIS_RETRY_COUNT') or 4 local REDIS_FIELD = "resources" local _M = {} @@ -51,14 +61,14 @@ local _M = {} ---------------------------- --- Initialize and connect to Redis --- @param host redis host --- @param port redis port --- @param password redis password (nil if no password) --- @param timeout redis timeout in milliseconds -function _M.init(host, port, password, timeout) +function _M.init() + local host = REDIS_HOST + local password = REDIS_PASS + local port = REDIS_PORT local redis = require "resty.redis" local red = redis:new() - red:set_timeout(timeout) + + red:set_timeout(REDIS_TIMEOUT) -- Connect to Redis server local retryCount = REDIS_RETRY_COUNT local connect, err = red:connect(host, port) @@ -204,7 +214,7 @@ function _M.generateResourceObj(ops, apiId, tenantObj, cors) resourceObj.operations[op].security = v.security end end - if cors then + if cors then resourceObj.cors = cors end if apiId then @@ -273,7 +283,7 @@ end --- Get all resource keys for a tenant in redis -- @param red redis client instance -- @param tenantId tenant id -function _M.getAllResourceKeys(red, tenantId) +function _M.getAllResources(red, tenantId) local keys, err = smembers(red, utils.concatStrings({"resources:", tenantId, ":__index__"})) if not keys then request.err(500, utils.concatStrings({"Failed to retrieve resource keys: ", err})) @@ -396,6 +406,24 @@ function _M.deleteSubscription(red, key) end end +----------------------------- +--- OAuth Tokens --- +----------------------------- +function _M.getOAuthToken(red, provider, token) + return get(red, utils.concatStrings({'oauth:providers:', provider, ':tokens:', token})) +end + + + +function _M.saveOAuthToken(red, provider, token, body, ttl) + set(red, utils.concatStrings({'oauth:providers:', provider, ':tokens:', token}), body) + if ttl ~= nil then + expire(red, utils.concatStrings({'oauth:providers:', provider, ':tokens:', token}), ttl) + end +end + + + --- Check health of gateway function _M.healthCheck() request.success(200, "Status: Gateway ready.") @@ -436,132 +464,138 @@ function _M.deleteSwagger(red, id) end end +function _M.setRateLimit(red, key, value, interval, expires) + return red:set(key, value, interval, expires) +end --- LRU Caching methods +function _M.getRateLimit(red, key) + return get(red, key) +end +-- LRU Caching methods -function exists(red, key) - if CACHING_ENABLED then +function exists(red, key) + if CACHING_ENABLED then local cached = c:get(key) - if cached ~= nil then + if cached ~= nil then return 1 - end + end -- if it isn't in the cache, try and load it in there local result = red:get(key) if result ~= ngx.null then c:set(key, result, CACHE_TTL) return 1 - end - return 0 - else - return red:exists(key) - end -end - -function get(red, key) - if CACHING_ENABLED then + end + return 0 + else + return red:exists(key) + end +end + +function get(red, key) + if CACHING_ENABLED then local cached, stale = c:get(key) if cached ~= nil then - return cached - else - local result = red:get(key) - c:set(key, result, CACHE_TTL) + return cached + else + local result = red:get(key) + c:set(key, result, CACHE_TTL) return result - end + end else return red:get(key) end end -function hget(red, key, id) - if CACHING_ENABLED then +function hget(red, key, id) + if CACHING_ENABLED then local cachedmap, stale = c:get(key) if cachedmap ~= nil then local cached = cachedmap:get(id) if cached ~= nil then - return cached + return cached else - local result = red:hget(key, id) - cachedmap:set(id, result, CACHE_TTL) + local result = red:hget(key, id) + cachedmap:set(id, result, CACHE_TTL) c:set(key, cachedmap, CACHE_TTL) return result end else local result = red:hget(key, id) - local newcache = lrucache.new(CACHE_SIZE) - newcache:set(id, result, CACHE_TTL) + local newcache = lrucache.new(CACHE_SIZE) + newcache:set(id, result, CACHE_TTL) c:set(key, newcache, CACHE_TTL) - return result + return result end else return red:hget(key, id) end -end +end -function hgetall(red, key) +function hgetall(red, key) return red:hgetall(key) -end +end function hset(red, key, id, value) - if CACHING_ENABLED then + if CACHING_ENABLED then local cachedmap = c:get(key) - if cachedmap ~= nil then - cachedmap:set(id, value, CACHE_TTL) + if cachedmap ~= nil then + cachedmap:set(id, value, CACHE_TTL) c:set(key, cachedmap, CACHE_TTL) return red:hset(key, id, value) - else + else local val = lrucache.new(CACHE_SIZE) - val:set(id, value, CACHE_TTL) + val:set(id, value, CACHE_TTL) c:set(key, val, CACHE_TTL) - end - end - return red:hset(key, id, value) -end - -function expire(red, key, ttl) - if CACHING_ENABLED then - local cached = c:get(key) - local value = '' - if cached ~= nil then -- just put it back in the cache with a ttl - value = cached - end + end + end + return red:hset(key, id, value) +end + +function expire(red, key, ttl) + if CACHING_ENABLED then + local cached = c:get(key) + local value = '' + if cached ~= nil then -- just put it back in the cache with a ttl + value = cached + end c:set(key, value, ttl) end return red:expire(key, ttl) -end +end -function del(red, key) - if CACHING_ENABLED then +function del(red, key) + if CACHING_ENABLED then c:delete(key) end return red:del(key) end - + function hdel(red, key, id) - if CACHING_ENABLED then - local cachecontents = c:get(key) + if CACHING_ENABLED then + local cachecontents = c:get(key) if cachecontents ~= nil then cachecontents:del(id) c:set(key, cachecontents, CACHE_TTL) - end + end end - return red:hdel(key, id) -end + return red:hdel(key, id) +end -function set(red, key, value) - return red:set(key, value) -end +function set(red, key, value) + return red:set(key, value) +end -function smembers(red, key) - return red:smembers(key) +function smembers(red, key) + return red:smembers(key) end function srem(red, key, id) - return red:srem(key, id) -end + return red:srem(key, id) +end function sadd(red, key, id) - return red:sadd(key, id) -end + return red:sadd(key, id) +end _M.get = get diff --git a/scripts/lua/management/lib/apis.lua b/scripts/lua/management/lib/apis.lua index 536036b..b42d70f 100644 --- a/scripts/lua/management/lib/apis.lua +++ b/scripts/lua/management/lib/apis.lua @@ -22,7 +22,6 @@ -- Module for querying APIs local cjson = require "cjson" -local redis = require "lib/redis" local utils = require "lib/utils" local request = require "lib/request" local resources = require "management/lib/resources" @@ -35,10 +34,10 @@ MANAGEDURL_PORT = (MANAGEDURL_PORT ~= nil and MANAGEDURL_PORT ~= '') and MANAGED local _M = {} --- Get all APIs in redis --- @param red redis client +-- @param ds dataStore.client -- @param queryParams object containing optional query parameters -function _M.getAllAPIs(red, queryParams) - local apis = redis.getAllAPIs(red) +function _M.getAllAPIs(dataStore, queryParams) + local apis = dataStore:getAllAPIs() local apiList if next(queryParams) ~= nil then apiList = filterAPIs(apis, queryParams); @@ -55,10 +54,10 @@ function _M.getAllAPIs(red, queryParams) end --- Get API by its id --- @param red redis client +-- @param ds dataStore.client -- @param id of API -function _M.getAPI(red, id) - local api = redis.getAPI(red, id) +function _M.getAPI(dataStore, id) + local api = dataStore:getAPI(id) if api == nil then request.err(404, utils.concatStrings({"Unknown api id ", id})) end @@ -66,15 +65,15 @@ function _M.getAPI(red, id) end --- Get belongsTo relation tenant --- @param red redis client +-- @param ds dataStore.client -- @param id id of API -function _M.getAPITenant(red, id) - local api = redis.getAPI(red, id) +function _M.getAPITenant(dataStore, id) + local api = dataStore:getAPI(id) if api == nil then request.err(404, utils.concatStrings({"Unknown api id ", id})) end local tenantId = api.tenantId - local tenant = redis.getTenant(red, tenantId) + local tenant = dataStore:getTenant(tenantId) if tenant == nil then request.err(404, utils.concatStrings({"Unknown tenant id ", tenantId})) end @@ -82,10 +81,10 @@ function _M.getAPITenant(red, id) end --- Add API to redis --- @param red redis client +-- @param ds dataStore.client -- @param decoded JSON body as a lua table -- @param existingAPI optional existing API for updates -function _M.addAPI(red, decoded, existingAPI) +function _M.addAPI(dataStore, decoded, existingAPI) -- Format basePath local basePath = decoded.basePath:sub(1,1) == '/' and decoded.basePath:sub(2) or decoded.basePath basePath = basePath:sub(-1) == '/' and basePath:sub(1, -2) or basePath @@ -95,7 +94,7 @@ function _M.addAPI(red, decoded, existingAPI) if basePath:sub(1,1) ~= '' then managedUrl = utils.concatStrings({managedUrl, "/", basePath}) end - local tenantObj = redis.getTenant(red, decoded.tenantId) + local tenantObj = dataStore:getTenant(decoded.tenantId) local managedUrlObj = { id = uuid, name = decoded.name, @@ -107,33 +106,34 @@ function _M.addAPI(red, decoded, existingAPI) managedUrl = managedUrl } -- Add API object to redis - managedUrlObj = redis.addAPI(red, uuid, managedUrlObj, existingAPI) + managedUrlObj = dataStore:addAPI(uuid, managedUrlObj, existingAPI) -- Add resources to redis for path, resource in pairs(decoded.resources) do local gatewayPath = utils.concatStrings({basePath, path}) gatewayPath = (gatewayPath:sub(1,1) == '/') and gatewayPath:sub(2) or gatewayPath resource.apiId = uuid - resources.addResource(red, resource, gatewayPath, tenantObj) + print('creating resource: ' .. cjson.encode(resource) .. 'path: ' .. gatewayPath .. 'tenantobj' .. cjson.encode(tenantObj)) + resources.addResource(dataStore, resource, gatewayPath, tenantObj) end return managedUrlObj end --- Delete API from gateway --- @param red redis client +-- @param ds dataStore.client -- @param id id of API to delete -function _M.deleteAPI(red, id) - local api = redis.getAPI(red, id) +function _M.deleteAPI(dataStore, id) + local api = dataStore:getAPI(id) if api == nil then request.err(404, utils.concatStrings({"Unknown API id ", id})) end -- Delete API - redis.deleteAPI(red, id) + dataStore:deleteAPI(id) -- Delete all resources for the API local basePath = api.basePath:sub(2) for path in pairs(api.resources) do local gatewayPath = utils.concatStrings({basePath, path}) gatewayPath = (gatewayPath:sub(1,1) == '/') and gatewayPath:sub(2) or gatewayPath - resources.deleteResource(red, gatewayPath, api.tenantId) + resources.deleteResource(dataStore, gatewayPath, api.tenantId) end return {} end diff --git a/scripts/lua/management/lib/resources.lua b/scripts/lua/management/lib/resources.lua index 8fb3669..041c481 100644 --- a/scripts/lua/management/lib/resources.lua +++ b/scripts/lua/management/lib/resources.lua @@ -21,9 +21,8 @@ --- @module resources -- Management interface for resources for the gateway -local redis = require "lib/redis" local utils = require "lib/utils" - +local cjson = require "cjson" local REDIS_FIELD = "resources" local _M = {} @@ -32,27 +31,27 @@ local _M = {} -- @param resource -- @param gatewayPath -- @param tenantObj -function _M.addResource(red, resource, gatewayPath, tenantObj) +function _M.addResource(dataStore, resource, gatewayPath, tenantObj) -- Create resource object and add to redis local redisKey = utils.concatStrings({"resources:", tenantObj.id, ":", gatewayPath}) local operations = resource.operations local apiId = resource.apiId local cors = resource.cors - local resourceObj = redis.generateResourceObj(operations, apiId, tenantObj, cors) - redis.createResource(red, redisKey, REDIS_FIELD, resourceObj) + local resourceObj = dataStore:generateResourceObj(operations, apiId, tenantObj, cors) + dataStore:createResource(redisKey, REDIS_FIELD, resourceObj) local indexKey = utils.concatStrings({"resources:", tenantObj.id, ":__index__"}) - redis.addResourceToIndex(red, indexKey, redisKey) + dataStore:addResourceToIndex(indexKey, redisKey) end --- Helper function for deleting resource in redis and appropriate conf files --- @param red redis instance +-- @param ds redis instance -- @param gatewayPath path in gateway -- @param tenantId tenant id -function _M.deleteResource(red, gatewayPath, tenantId) +function _M.deleteResource(dataStore, gatewayPath, tenantId) local redisKey = utils.concatStrings({"resources:", tenantId, ":", gatewayPath}) - redis.deleteResource(red, redisKey, REDIS_FIELD) + dataStore:deleteResource(redisKey, REDIS_FIELD) local indexKey = utils.concatStrings({"resources:", tenantId, ":__index__"}) - redis.deleteResourceFromIndex(red, indexKey, redisKey) + dataStore:deleteResourceFromIndex(indexKey, redisKey) end return _M diff --git a/scripts/lua/management/lib/tenants.lua b/scripts/lua/management/lib/tenants.lua index 97098db..5749c5f 100644 --- a/scripts/lua/management/lib/tenants.lua +++ b/scripts/lua/management/lib/tenants.lua @@ -29,7 +29,7 @@ local apis = require "management/lib/apis" local _M = {}; -function _M.addTenant(red, decoded, existingTenant) +function _M.addTenant(dataStore, decoded, existingTenant) -- Return tenant object local uuid = existingTenant ~= nil and existingTenant.id or utils.uuid() local tenantObj = { @@ -37,15 +37,15 @@ function _M.addTenant(red, decoded, existingTenant) namespace = decoded.namespace, instance = decoded.instance } - tenantObj = redis.addTenant(red, uuid, tenantObj) + tenantObj = dataStore:addTenant(uuid, tenantObj) return cjson.decode(tenantObj) end --- Get all tenants in redis --- @param red redis client +-- @param ds redis client -- @param queryParams object containing optional query parameters -function _M.getAllTenants(red, queryParams) - local tenants = redis.getAllTenants(red) +function _M.getAllTenants(dataStore, queryParams) + local tenants = dataStore:getAllTenants() local tenantList if next(queryParams) ~= nil then tenantList = filterTenants(tenants, queryParams); @@ -86,10 +86,10 @@ function filterTenants(tenants, queryParams) end --- Get tenant by its id --- @param red redis client +-- @param ds redis client -- @param id tenant id -function _M.getTenant(red, id) - local tenant = redis.getTenant(red, id) +function _M.getTenant(dataStore, id) + local tenant = dataStore:getTenant(id) if tenant == nil then request.err(404, utils.concatStrings({"Unknown tenant id ", id })) end @@ -97,11 +97,11 @@ function _M.getTenant(red, id) end --- Get APIs associated with tenant --- @param red redis client +-- @param ds redis client -- @param id tenant id -- @param queryParams object containing optional query parameters -function _M.getTenantAPIs(red, id, queryParams) - local apis = redis.getAllAPIs(red) +function _M.getTenantAPIs(dataStore, id, queryParams) + local apis = dataStore:getAllAPIs() local apiList if next(queryParams) ~= nil then apiList = filterTenantAPIs(id, apis, queryParams); @@ -148,14 +148,14 @@ function filterTenantAPIs(id, apis, queryParams) end --- Delete tenant from gateway --- @param red redis client +-- @param ds redis client -- @param id id of tenant to delete -function _M.deleteTenant(red, id) - local tenantAPIs = _M.getTenantAPIs(red, id, {}) +function _M.deleteTenant(dataStore, id) + local tenantAPIs = _M.getTenantAPIs(dataStore, id, {}) for _, v in pairs(tenantAPIs) do - apis.deleteAPI(red, v.id) + apis.deleteAPI(dataStore, v.id) end - redis.deleteTenant(red, id) + dataStore:deleteTenant(id) return {} end diff --git a/scripts/lua/management/lib/validation.lua b/scripts/lua/management/lib/validation.lua index 17af88a..96686d8 100644 --- a/scripts/lua/management/lib/validation.lua +++ b/scripts/lua/management/lib/validation.lua @@ -22,15 +22,14 @@ -- Module for validating api body local cjson = require "cjson" -local redis = require "lib/redis" local utils = require "lib/utils" local _M = {} -function _M.validate(red, decoded) +function _M.validate(dataStore, decoded) local fields = {"name", "basePath", "tenantId", "resources"} for _, v in pairs(fields) do - local res, err = isValid(red, v, decoded[v]) + local res, err = isValid(dataStore, v, decoded[v]) if res == false then return err end @@ -39,10 +38,10 @@ function _M.validate(red, decoded) end --- Check JSON body fields for errors --- @param red Redis client instance +-- @param ds edis client instance -- @param field name of field -- @param object field object -function isValid(red, field, object) +function isValid(dataStore, field, object) -- Check that field exists in body if not object then return false, { statusCode = 400, message = utils.concatStrings({"Missing field '", field, "' in request body."}) } @@ -56,7 +55,7 @@ function isValid(red, field, object) end -- Additional check for tenantId if field == "tenantId" then - local tenant = redis.getTenant(red, object) + local tenant = dataStore:getTenant(object) if tenant == nil then return false, { statusCode = 404, message = utils.concatStrings({"Unknown tenant id ", object }) } end @@ -152,4 +151,4 @@ function checkOptionalPolicies(policies, security) end end -return _M \ No newline at end of file +return _M diff --git a/scripts/lua/management/routes/apis.lua b/scripts/lua/management/routes/apis.lua index 42215aa..d862c3a 100644 --- a/scripts/lua/management/routes/apis.lua +++ b/scripts/lua/management/routes/apis.lua @@ -22,7 +22,7 @@ -- Management interface for apis for the gateway local cjson = require "cjson" -local redis = require "lib/redis" +local dataStore = require "lib/dataStore" local utils = require "lib/utils" local request = require "lib/request" local apis = require "management/lib/apis" @@ -37,84 +37,82 @@ local REDIS_PASS = os.getenv("REDIS_PASS") local _M = {} --- Request handler for routing API calls appropriately -function _M.requestHandler() +function _M.requestHandler(dataStore) local requestMethod = ngx.req.get_method() ngx.header.content_type = "application/json; charset=utf-8" if requestMethod == "GET" then - getAPIs() + getAPIs(dataStore) elseif requestMethod == 'POST' or requestMethod == 'PUT' then - addAPI() + addAPI(dataStore) elseif requestMethod == "DELETE" then - deleteAPI() + deleteAPI(dataStore) else request.err(400, "Invalid verb.") end end -function getAPIs() +function getAPIs(dataStore) local queryParams = ngx.req.get_uri_args() local id = ngx.var.api_id local version = ngx.var.version - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) if id == '' then local apiList if version == 'v1' then - apiList = apis.getAllAPIs(red, queryParams) + apiList = apis.getAllAPIs(dataStore, queryParams) elseif version == 'v2' then local tenantId = ngx.var.tenantId local v2ApiList = {} - apiList = tenants.getTenantAPIs(red, tenantId, queryParams) + apiList = tenants.getTenantAPIs(dataStore, tenantId, queryParams) for _, api in pairs(apiList) do v2ApiList[#v2ApiList+1] = { artifact_id = api.id, managed_url = api.managedUrl, - open_api_doc = redis.getSwagger(red, api.id) + open_api_doc = dataStore:getSwagger(api.id) } end apiList = v2ApiList end apiList = (next(apiList) == nil) and "[]" or cjson.encode(apiList) - redis.close(red) + dataStore:close(ds) request.success(200, apiList) else local query = ngx.var.query if query ~= '' then if query ~= "tenant" then - redis.close(red) + dataStore:close() request.err(400, "Invalid request") else - local tenant = apis.getAPITenant(red, id) + local tenant = apis.getAPITenant(dataStore, id) tenant = cjson.encode(tenant) - redis.close(red) + dataStore:close() request.success(200, tenant) end else - local api = apis.getAPI(red, id) + local api = apis.getAPI(dataStore, id) if version == 'v1' then - redis.close(red) + dataStore:close() api = cjson.encode(api) request.success(200, api) elseif version == 'v2' then local returnObj = { artifact_id = api.id, managed_url = api.managedUrl, - open_api_doc = redis.getSwagger(red, api.id) + open_api_doc = dataStore:getSwagger(ds, api.id) } - redis.close(red) + dataStore.close() request.success(200, cjson.encode(returnObj)) end end end end -function addAPI() - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) +function addAPI(dataStore) local id = ngx.var.api_id - local existingAPI = checkForExistingAPI(red, id) + local existingAPI = checkForExistingAPI(dataStore, id) ngx.req.read_body() local args = ngx.req.get_body_data() if not args then - redis.close(red) + dataStore:close() request.err(400, "Missing request body") end -- Convert json into Lua table @@ -131,54 +129,53 @@ function addAPI() namespace = tenantId, instance = '' } - tenants.addTenant(red, tenant, {id = tenantId}) + tenants.addTenant(dataStore, tenant, {id = tenantId}) decoded.tenantId = tenantId end -- Check for api id in JSON body if existingAPI == nil and decoded.id ~= nil then - existingAPI = redis.getAPI(red, decoded.id) + existingAPI = dataStore.getAPI(decoded.id) if existingAPI == nil then - redis.close(red) + dataStore:close() request.err(404, utils.concatStrings({"Unknown API id ", decoded.id})) end end - local err = validation.validate(red, decoded) + local err = validation.validate(dataStore, decoded) if err ~= nil then - redis.close(red) + dataStore:close() request.err(err.statusCode, err.message) end - local managedUrlObj = apis.addAPI(red, decoded, existingAPI) + local managedUrlObj = apis.addAPI(dataStore, decoded, existingAPI) if version == 'v1' then - redis.close(red) + dataStore:close(ds) managedUrlObj = cjson.encode(managedUrlObj) request.success(200, managedUrlObj) elseif version == 'v2' then - redis.addSwagger(red, managedUrlObj.id, swaggerTable) + dataStore:addSwagger(managedUrlObj.id, swaggerTable) local returnObj = { artifact_id = managedUrlObj.id, managed_url = managedUrlObj.managedUrl, open_api_doc = swaggerTable } - redis.close(red) + dataStore:close() request.success(200, cjson.encode(returnObj)) end end -function deleteAPI() - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) +function deleteAPI(dataStore) local id = ngx.var.api_id if id == nil or id == '' then - redis.close(red) + dataStore:close() request.err(400, "No id specified.") end - apis.deleteAPI(red, id) + apis.deleteAPI(ds, id) local version = ngx.var.version if version == 'v1' then - redis.close(red) + dataStore:close() request.success(200, cjson.encode({})) elseif version == 'v2' then - redis.deleteSwagger(red, id) - redis.close(red) + dataStore:deleteSwagger(id) + dataStore:close() request.success(204) end end @@ -186,12 +183,12 @@ end --- Check for api id from uri and use existing API if it already exists in redis -- @param red Redis client instance -- @param id API id to check -function checkForExistingAPI(red, id) +function checkForExistingAPI(dataStore, id) local existing if id ~= nil and id ~= '' then - existing = redis.getAPI(red, id) + existing = dataStore:getAPI(id) if existing == nil then - redis.close(red) + dataStore:close() request.err(404, utils.concatStrings({"Unknown API id ", id})) end end diff --git a/scripts/lua/management/routes/subscriptions.lua b/scripts/lua/management/routes/subscriptions.lua index e9cc9db..40b5986 100644 --- a/scripts/lua/management/routes/subscriptions.lua +++ b/scripts/lua/management/routes/subscriptions.lua @@ -45,7 +45,7 @@ function _M.requestHandler() end --- v2 -- +-- v2 -- function v2() local requestMethod = ngx.req.get_method() @@ -187,4 +187,4 @@ function validateSubscriptionBody() return redisKey end -return _M \ No newline at end of file +return _M diff --git a/scripts/lua/management/routes/tenants.lua b/scripts/lua/management/routes/tenants.lua index 7f3f7f3..9f79727 100644 --- a/scripts/lua/management/routes/tenants.lua +++ b/scripts/lua/management/routes/tenants.lua @@ -33,65 +33,62 @@ local REDIS_PASS = os.getenv("REDIS_PASS") local _M = {}; --- Request handler for routing tenant calls appropriately -function _M.requestHandler() +function _M.requestHandler(dataStore) local requestMethod = ngx.req.get_method() ngx.header.content_type = "application/json; charset=utf-8" if requestMethod == "GET" then - getTenants() + getTenants(dataStore) elseif requestMethod == "PUT" or requestMethod == "POST" then - addTenant() + addTenant(dataStore) elseif requestMethod == "DELETE" then - deleteTenant() + deleteTenant(dataStore) else request.err(400, "Invalid verb.") end end -function addTenant() +function addTenant(dataStore) -- Open connection to redis or use one from connection pool - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) -- Check for tenant id and use existingTenant if it already exists in redis - local existingTenant = checkForExistingTenant(red) + local existingTenant = checkForExistingTenant(dataStore) -- Read in the PUT JSON Body ngx.req.read_body() local args = ngx.req.get_body_data() if not args then - redis.close(red) + dataStore:close() request.err(400, "Missing request body") end -- Convert json into Lua table local decoded = cjson.decode(args) -- Check for tenant id in JSON body if existingTenant == nil and decoded.id ~= nil then - existingTenant = redis.getTenant(red, decoded.id) + existingTenant = dataStore:getTenant(decoded.id) if existingTenant == nil then - redis.close(red) request.err(404, utils.concatStrings({"Unknown Tenant id ", decoded.id})) end end -- Error checking local res, err = utils.tableContainsAll(decoded, {"namespace", "instance"}) if res == false then - redis.close(red) + dataStore:close() request.err(err.statusCode, err.message) end -- Return tenant object - local tenantObj = tenants.addTenant(red, decoded, existingTenant) + local tenantObj = tenants.addTenant(dataStore, decoded, existingTenant) tenantObj = cjson.encode(tenantObj) - redis.close(red) request.success(200, tenantObj) end --- Check for tenant id from uri and use existing tenant if it already exists in redis -- @param red Redis client instance -function checkForExistingTenant(red) +function checkForExistingTenant(dataStore) local id = ngx.var.tenant_id local existing -- Get object from redis if id ~= nil and id ~= '' then - existing = redis.getTenant(red, id) + existing = dataStore:getTenant(id) if existing == nil then - redis.close(red) + dataStore:close() request.err(404, utils.concatStrings({"Unknown Tenant id ", id})) end end @@ -100,31 +97,30 @@ end --- Get one or all tenants from the gateway -- GET /v1/tenants -function getTenants() +function getTenants(dataStore) local queryParams = ngx.req.get_uri_args() local id = ngx.var.tenant_id - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) if id == '' then - local tenantList = tenants.getAllTenants(red, queryParams) + local tenantList = tenants.getAllTenants(dataStore, queryParams) tenantList = (next(tenantList) == nil) and "[]" or cjson.encode(tenantList) - redis.close(red) + dataStore:close() request.success(200, tenantList) else local query = ngx.var.query if query ~= '' then if query ~= "apis" then - redis.close(red) + dataStore:close() request.err(400, "Invalid request") else - local apiList = tenants.getTenantAPIs(red, id, queryParams) + local apiList = tenants.getTenantAPIs(dataStore, id, queryParams) apiList = (next(apiList) == nil) and "[]" or cjson.encode(apiList) - redis.close(red) + dataStore:close() request.success(200, apiList) end else - local tenant = tenants.getTenant(red, id) + local tenant = tenants.getTenant(dataStore, id) tenant = cjson.encode(tenant) - redis.close(red) + dataStore:close() request.success(200, tenant) end end @@ -132,19 +128,17 @@ end --- Delete tenant from gateway -- DELETE /v1/tenants/<id> -function deleteTenant() +function deleteTenant(dataStore) local id = ngx.var.tenant_id if id == nil or id == '' then request.err(400, "No id specified.") end - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) - local tenant = redis.getTenant(red, id) + local tenant = dataStore:getTenant(id) if tenant == nil then - redis.close(red) + dataStore:close() request.err(404, utils.concatStrings({"Unknown tenant id ", id})) end - tenants.deleteTenant(red, id) - redis.close(red) + tenants.deleteTenant(dataStore, id) request.success(200, cjson.encode({})) end diff --git a/scripts/lua/oauth/facebook.lua b/scripts/lua/oauth/facebook.lua index a739ca9..d1a5a81 100644 --- a/scripts/lua/oauth/facebook.lua +++ b/scripts/lua/oauth/facebook.lua @@ -20,9 +20,9 @@ local request = require 'lib/request' local cjson = require 'cjson' local utils = require "lib/utils" -local redis = require "lib/redis" -function validateOAuthToken (red, token) +local _M = {} +function _M.process(dataStore, token) local headerName = utils.concatStrings({'http_', 'x-facebook-app-token'}):gsub("-", "_") @@ -32,19 +32,19 @@ function validateOAuthToken (red, token) return nil end - local key = utils.concatStrings({'oauth:providers:facebook:tokens:', token}) - if redis.exists(red, key) == 1 then - return cjson.decode(redis.get(red,key)) + local result = dataStore:getOAuthToken('facebook', token) + if result ~= ngx.null then + return cjson.decode(result) end - - local key = utils.concatStrings({'oauth:providers:facebook:tokens:', token, facebookAppToken}) - if redis.exists(red, key) == 1 then - return cjson.decode(redis.get(red, key)) + result = dataStore:getOAuthToken('facebook', utils.concatStrings({token, facebookAppToken})) + if result ~= ngx.null then + return cjson.decode(result) end - return exchangeOAuthToken(red, token, facebookAppToken) + + return exchangeOAuthToken(dataStore, token, facebookAppToken) end -function exchangeOAuthToken(red, token, facebookAppToken) +function exchangeOAuthToken(dataStore, token, facebookAppToken) local http = require 'resty.http' local request = require "lib/request" local httpc = http.new() @@ -70,14 +70,10 @@ function exchangeOAuthToken(red, token, facebookAppToken) if (json_resp['error']) then return nil end - -- facebook uses a different expire field than others - json_resp['expires'] = json_resp['expires_at'] -- convert Facebook's response -- Read more about the fields at: https://developers.google.com/identity/protocols/OpenIDConnect#obtainuserinfo - redis.set(red, key, cjson.encode(json_resp)) - redis.expire(red, json_resp, json_resp['expires']) + dataStore:saveOAuthToken('facebook', utils.concatStrings({token, facebookAppToken}), cjson.encode(json_resp), json_resp['expires_at']) return json_resp end -return validateOAuthToken - +return _M diff --git a/scripts/lua/oauth/github.lua b/scripts/lua/oauth/github.lua index 9d2c208..2335477 100644 --- a/scripts/lua/oauth/github.lua +++ b/scripts/lua/oauth/github.lua @@ -25,13 +25,13 @@ local redis = require "lib/redis" local request = require "lib/request" local cjson = require "cjson" local utils = require "lib/utils" +local _M = {} -function validateOAuthToken (red, token) - local httpc = http.new() - local key = utils.concatStrings({'oauth:provider:github:', token}) - if redis.exists(red, key) == 1 then - return cjson.decode(redis.get(red, key)) - end +function _M.process(dataStore, token) + local result = dataStore:getOAuthToken('github', token) + if result ~= ngx.null then + return cjson.decode(result) + end local request_options = { headers = { @@ -40,6 +40,8 @@ function validateOAuthToken (red, token) ssl_verify = false } + local httpc = http.new() + local envUrl = os.getenv('TOKEN_GITHUB_URL') envUrl = envUrl ~= nil and envUrl or 'https://api.github.com/user' local request_uri = utils.concatStrings({envUrl, '?access_token=', token}) @@ -60,12 +62,10 @@ function validateOAuthToken (red, token) return nil end - redis.set(red, key, cjson.encode(json_resp)) + dataStore:saveOAuthToken('github', token) -- convert Github's response -- Read more about the fields at: https://developers.google.com/identity/protocols/OpenIDConnect#obtainuserinfo return json_resp end -return validateOAuthToken - - +return _M diff --git a/scripts/lua/oauth/google.lua b/scripts/lua/oauth/google.lua index aba85ba..bea1f6c 100644 --- a/scripts/lua/oauth/google.lua +++ b/scripts/lua/oauth/google.lua @@ -25,12 +25,15 @@ local request = require "lib/request" local utils = require "lib/utils" local redis = require "lib/redis" -function validateOAuthToken (red, token) +local _M = {} +function _M.process (ds, token) + + local result = dataStore:getOAuthToken(dataStore, 'google', token) + local httpc = http.new() - local key = utils.concatStrings({'oauth:provider:google:', token}) - if redis.exists(red, key) == 1 then - return cjson.decode(redis.get(red, key)) - end + if result ~= ngx.null then + return cjson.decode(result) + end local request_options = { headers = { @@ -38,6 +41,7 @@ function validateOAuthToken (red, token) }, ssl_verify = false } + local envUrl = os.getenv('TOKEN_GOOGLE_URL') envUrl = envUrl ~= nil and envUrl or 'https://www.googleapis.com/oauth2/v3/tokeninfo' local request_uri = utils.concatStrings({envUrl, "?access_token=", token}) @@ -48,19 +52,15 @@ function validateOAuthToken (red, token) request.err(500, 'OAuth provider error.') return nil end - local json_resp = cjson.decode(res.body) - - if (json_resp.error_description) then + local json_resp = cjson.decode(res.body) + if json_resp['error_description'] ~= nil then return nil end - - redis.set(red, key, cjson.encode(json_resp)) - redis.expire(red, key, json_resp['expires']) + + dataStore:saveOAuthToken('google', token, cjson.encode(json_resp), json_resp['expires']) -- convert Google's response -- Read more about the fields at: https://developers.google.com/identity/protocols/OpenIDConnect#obtainuserinfo return json_resp end -return validateOAuthToken - - +return _M diff --git a/scripts/lua/oauth/mock.lua b/scripts/lua/oauth/mock.lua index 1ddb8a0..2b88511 100644 --- a/scripts/lua/oauth/mock.lua +++ b/scripts/lua/oauth/mock.lua @@ -21,7 +21,8 @@ --- -- A fake oauth provider for testing local cjson = require "cjson" -function validateOAuthToken (red, token) +local _M = {} +function _M.process (red, token) local result if token == "test" then local goodResult = [[ @@ -36,4 +37,4 @@ function validateOAuthToken (red, token) end end -return validateOAuthToken +return _M diff --git a/scripts/lua/policies/rateLimit.lua b/scripts/lua/policies/rateLimit.lua index 22005c5..082824a 100644 --- a/scripts/lua/policies/rateLimit.lua +++ b/scripts/lua/policies/rateLimit.lua @@ -29,7 +29,7 @@ local _M = {} -- @param red redis client instance -- @param obj rateLimit object containing interval, rate, scope, subscription fields -- @param apiKey optional api key to use if subscription is set to true -function limit(red, obj, apiKey) +function limit(dataStore, obj, apiKey) local rate = obj.interval / obj.rate local tenantId = ngx.var.tenant local gatewayPath = ngx.var.gatewayPath @@ -47,8 +47,9 @@ function limit(red, obj, apiKey) if obj.subscription ~= nil and obj.subscription == true and apiKey ~= nil then k = utils.concatStrings({k, ":subscription:", apiKey}) end - if red:get(k) == ngx.null then - red:set(k, "", "PX", math.floor(rate*1000)) + + if dataStore:getRateLimit(k) == ngx.null then + dataStore:setRateLimit(k, "", "PX", math.floor(rate*1000)) else return request.err(429, 'Rate limit exceeded') end diff --git a/scripts/lua/policies/security.lua b/scripts/lua/policies/security.lua index 1f5b490..0761f34 100644 --- a/scripts/lua/policies/security.lua +++ b/scripts/lua/policies/security.lua @@ -28,13 +28,13 @@ local request = require "lib/request" local utils = require "lib/utils" --- Allow or block a request by calling a loaded security policy -- @param securityObj an object out of the security array in a given tenant / api / resource -function process(securityObj) +function process(dataStore, securityObj) local ok, result = pcall(require, utils.concatStrings({'policies/security/', securityObj.type})) if not ok then ngx.log(ngx.ERR, 'An unexpected error ocurred while processing the security policy: ' .. securityObj.type) request.err(500, 'Gateway error.') end - return result.process(securityObj) + return result.process(dataStore, securityObj) end -- Wrap process in code to load the correct module diff --git a/scripts/lua/policies/security/apiKey.lua b/scripts/lua/policies/security/apiKey.lua index f366292..1336243 100644 --- a/scripts/lua/policies/security/apiKey.lua +++ b/scripts/lua/policies/security/apiKey.lua @@ -23,7 +23,7 @@ -- Check a subscription with an API Key -- @author Cody Walker (cmwalker), Alex Song (songs) -local redis = require "lib/redis" +local dataStore = require "lib/dataStore" local utils = require "lib/utils" local request = require "lib/request" @@ -41,7 +41,7 @@ local _M = {} -- @param scope scope of the subscription -- @param apiKey the subscription api key -- @param return boolean value indicating if the subscription exists in redis -function validate(red, tenant, gatewayPath, apiId, scope, apiKey) +function validate(dataStore, tenant, gatewayPath, apiId, scope, apiKey) -- Open connection to redis or use one from connection pool local k if scope == 'tenant' then @@ -52,18 +52,15 @@ function validate(red, tenant, gatewayPath, apiId, scope, apiKey) k = utils.concatStrings({'subscriptions:tenant:', tenant, ':api:', apiId}) end k = utils.concatStrings({k, ':key:', apiKey}) - if redis.exists(red, k) == 1 then - return k - else - return nil + if dataStore:exists(k) == 1 then + return k + else + return nil end end -function process(securityObj) - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 1000) - local result = processWithRedis(red, securityObj, sha256) - redis.close(red) - return result +function process(dataStore, securityObj) + return processWithHashFunction(dataStore, securityObj, sha256) end --- Process the security object @@ -71,10 +68,10 @@ end -- @param securityObj security object from nginx conf file -- @param hashFunction a function that will be called to hash the string -- @return apiKey api key for the subscription -function processWithRedis(red, securityObj, hashFunction) +function processWithHashFunction(dataStore, securityObj, hashFunction) local tenant = ngx.var.tenant local gatewayPath = ngx.var.gatewayPath - local apiId = redis.resourceToApi(red, utils.concatStrings({'resources:', tenant, ':', gatewayPath})) + local apiId = dataStore:resourceToApi(utils.concatStrings({'resources:', tenant, ':', gatewayPath})) local scope = securityObj.scope local header = (securityObj.header == nil) and 'x-api-key' or securityObj.header local apiKey = ngx.var[utils.concatStrings({'http_', header}):gsub("-", "_")] @@ -85,11 +82,11 @@ function processWithRedis(red, securityObj, hashFunction) if securityObj.hashed then apiKey = hashFunction(apiKey) end - local key = validate(red, tenant, gatewayPath, apiId, scope, apiKey) + local key = validate(dataStore, tenant, gatewayPath, apiId, scope, apiKey) if key == nil then - request.err(401, 'Unauthorized') + request.err(401, 'Unauthorized') return nil - end + end ngx.var.apiKey = apiKey return apiKey end @@ -105,8 +102,7 @@ function sha256(str) local digest = sha:final() return resty_str.to_hex(digest) end - +_M.processWithHashFunction = processWithHashFunction _M.process = process -_M.processWithRedis = processWithRedis return _M diff --git a/scripts/lua/policies/security/clientSecret.lua b/scripts/lua/policies/security/clientSecret.lua index 787a1cd..85224d7 100644 --- a/scripts/lua/policies/security/clientSecret.lua +++ b/scripts/lua/policies/security/clientSecret.lua @@ -1,6 +1,6 @@ -- Copyright (c) 2016 IBM. All rights reserved. -- --- +-- -- Permission is hereby granted, free of charge, to any person obtaining a -- copy of this software and associated documentation files (the "Software"), -- to deal in the Software without restriction, including without limitation @@ -23,7 +23,7 @@ -- Check a subscription with a client id and a hashed secret local _M = {} -local redis = require "lib/redis" +local dataStore = require "lib/dataStore" local utils = require "lib/utils" local request = require "lib/request" @@ -33,52 +33,49 @@ local REDIS_PASS = os.getenv("REDIS_PASS") --- Process function main entry point for the security block. -- Takes 2 headers and decides if the request should be allowed based on a hashed secret key ---@param securityObj the security object loaded from nginx.conf +--@param securityObj the security object loaded from nginx.conf --@return a string representation of what this looks like in redis :clientsecret:clientid:hashed secret -function process(securityObj) - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 1000) - local result = processWithHashFunction(red, securityObj, utils.hash) - redis.close(red) - return result -end +function process(dataStore, securityObj) + local result = processWithHashFunction(dataStore, securityObj, utils.hash) +end --- In order to properly test this functionallity, I use this function to do all of the business logic with injected dependencies -- Takes 2 headers and decides if the request should be allowed based on a hashed secret key --- @param red the redis instance to perform the lookup on +-- @param red the redis instance to perform the lookup on -- @param securityObj the security configuration for the tenant/resource/api we are verifying -- @param hashFunction the function used to perform the hash of the api secret -function processWithHashFunction(red, securityObj, hashFunction) +function processWithHashFunction(dataStore, securityObj, hashFunction) -- pull the configuration from nginx local tenant = ngx.var.tenant local gatewayPath = ngx.var.gatewayPath local apiId = ngx.var.apiId local scope = securityObj.scope - + -- allow support for custom headers - local location = (securityObj.keyLocation == nil) and 'http_' or securityObj.keyLocation + local location = (securityObj.keyLocation == nil) and 'http_' or securityObj.keyLocation if location == 'header' then location = 'http_' end - + local clientIdName = (securityObj.idFieldName == nil) and 'X-Client-ID' or securityObj.idFieldName - + local clientId = ngx.var[utils.concatStrings({location, clientIdName}):gsub("-", "_")] -- if they didn't supply whatever header this is configured to require, error out if clientId == nil or clientId == '' then request.err(401, clientIdName .. " required") return false end - local clientSecretName = (securityObj.secretFieldName == nil) and 'X-Client-Secret' or securityObj.secretFieldName + local clientSecretName = (securityObj.secretFieldName == nil) and 'X-Client-Secret' or securityObj.secretFieldName _G.clientSecretName = clientSecretName:lower() - + local clientSecret = ngx.var[utils.concatStrings({location, clientSecretName}):gsub("-","_")] if clientSecret == nil or clientSecret == '' then request.err(401, clientSecretName .. " required") return false end --- hash the secret - local result = validate(red, tenant, gatewayPath, apiId, scope, clientId, hashFunction(clientSecret)) - if result == nil then +-- hash the secret + local result = validate(dataStore, tenant, gatewayPath, apiId, scope, clientId, hashFunction(clientSecret)) + if result == nil then request.err(401, "Secret mismatch or not subscribed to this api.") end ngx.var.apiKey = clientId @@ -86,13 +83,13 @@ function processWithHashFunction(red, securityObj, hashFunction) end --- Validate that the subscription exists in redis --- @param tenant the tenantId we are checking for +-- @param tenant the tenantId we are checking for -- @param gatewayPath the possible resource we are checking for --- @param apiId if we are checking for an api +-- @param apiId if we are checking for an api -- @param scope which values should we be using to find the location of the secret --- @param clientId the subscribed client id +-- @param clientId the subscribed client id -- @param clientSecret the hashed client secret -function validate(red, tenant, gatewayPath, apiId, scope, clientId, clientSecret) +function validate(dataStore, tenant, gatewayPath, apiId, scope, clientId, clientSecret) -- Open connection to redis or use one from connection pool local k if scope == 'tenant' then @@ -102,14 +99,14 @@ function validate(red, tenant, gatewayPath, apiId, scope, clientId, clientSecret elseif scope == 'api' then k = utils.concatStrings({'subscriptions:tenant:', tenant, ':api:', apiId}) end - -- using the same key location in redis, just using :clientsecret: instead of :key: + -- using the same key location in redis, just using :clientsecret: instead of :key: k = utils.concatStrings({k, ':clientsecret:', clientId, ':', clientSecret}) - if redis.exists(red, k) == 1 then - return k - else + if dataStore:exists(k) == 1 then + return k + else return nil - end + end end _M.processWithHashFunction = processWithHashFunction _M.process = process -return _M +return _M diff --git a/scripts/lua/policies/security/oauth2.lua b/scripts/lua/policies/security/oauth2.lua index b799d3b..ecd62cf 100644 --- a/scripts/lua/policies/security/oauth2.lua +++ b/scripts/lua/policies/security/oauth2.lua @@ -26,7 +26,7 @@ local utils = require "lib/utils" local request = require "lib/request" -local redis = require "lib/redis" +local dataStore = require "lib/dataStore" local cjson = require "cjson" local REDIS_HOST = os.getenv("REDIS_HOST") @@ -35,26 +35,20 @@ local REDIS_PASS = os.getenv("REDIS_PASS") local _M = {} -function process(securityObj) - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 1000) - local result = processWithRedis(red, securityObj) - redis.close(red) - return result -end -- Process the security object -- @param securityObj security object from nginx conf file -- @return oauthId oauth identification -function processWithRedis(red, securityObj) +function process(dataStore, securityObj) local accessToken = ngx.var['http_Authorization'] if accessToken == nil then request.err(401, "No Authorization header provided") return nil end accessToken = string.gsub(accessToken, '^Bearer%s', '') - - local token = exchangeWithRedis(red, accessToken, securityObj.provider) + + local token = exchange(dataStore, accessToken, securityObj.provider) if token == nil then request.err(401, 'Token didn\'t work or provider doesn\'t support OpenID connect. ') return nil @@ -72,20 +66,22 @@ end -- @param token the accessToken passed in the authorization header of the routing request -- @param provider the name of the provider we will load from a file. Currently supported google/github/facebook -- @return the json object recieved from exchanging tokens with the provider - function exchangeWithRedis(red, token, provider) +function exchange(dataStore, token, provider) -- exchange tokens with the provider - local loaded, provider = pcall(require, utils.concatStrings({'oauth/', provider})) - + local loaded, impl = pcall(require, utils.concatStrings({'oauth/', provider})) if not loaded then request.err(500, 'Error loading OAuth provider authentication module') print("error loading provider.") return nil end - local token = provider(red, token) + + local result = impl.process(dataStore, token) + if result == nil then + request.err('401', 'OAuth token didn\'t work or provider doesn\'t support OpenID connect') + end -- cache the token - return token + return result end _M.process = process -_M.processWithRedis = processWithRedis return _M diff --git a/scripts/lua/routing.lua b/scripts/lua/routing.lua index fdb576c..cd1d11b 100644 --- a/scripts/lua/routing.lua +++ b/scripts/lua/routing.lua @@ -25,7 +25,6 @@ local cjson = require "cjson" local url = require "url" local utils = require "lib/utils" local request = require "lib/request" -local redis = require "lib/redis" -- load policies local security = require "policies/security" local mapping = require "policies/mapping" @@ -33,16 +32,12 @@ local rateLimit = require "policies/rateLimit" local backendRouting = require "policies/backendRouting" local cors = require "cors" -local REDIS_HOST = os.getenv("REDIS_HOST") -local REDIS_PORT = os.getenv("REDIS_PORT") -local REDIS_PASS = os.getenv("REDIS_PASS") local _M = {} --- Main function that handles parsing of invocation details and carries out implementation -function _M.processCall() +function _M.processCall(dataStore) -- Get resource object from redis - local red = redis.init(REDIS_HOST, REDIS_PORT, REDIS_PASS, 10000) local tenantId = ngx.var.tenant local gatewayPath = ngx.var.gatewayPath local i, j = ngx.var.request_uri:find("/api/([^/]+)") @@ -50,12 +45,12 @@ function _M.processCall() if ngx.req.get_headers()["x-debug-mode"] == "true" then setRequestLogs() end - local resourceKeys = redis.getAllResourceKeys(red, tenantId) - local redisKey = _M.findRedisKey(resourceKeys, tenantId, gatewayPath) + local resourceKeys = dataStore:getAllResources(tenantId) + local redisKey = _M.findResource(resourceKeys, tenantId, gatewayPath) if redisKey == nil then request.err(404, 'Not found.') end - local obj = cjson.decode(redis.getResource(red, redisKey, "resources")) + local obj = cjson.decode(dataStore:getResource(redisKey, "resources")) cors.processCall(obj) ngx.var.tenantNamespace = obj.tenantNamespace ngx.var.tenantInstance = obj.tenantInstance @@ -66,7 +61,7 @@ function _M.processCall() local key if (opFields.security) then for _, sec in ipairs(opFields.security) do - local result = security.process(sec) + local result = security.process(dataStore, sec) if key == nil and sec.type ~= "oauth2" then key = result -- use key from either apiKey or clientSecret security policy end @@ -80,13 +75,13 @@ function _M.processCall() backendRouting.setRoute(opFields.backendUrl) -- Parse policies if opFields.policies ~= nil then - parsePolicies(red, opFields.policies, key) + parsePolicies(dataStore, opFields.policies, key) end -- Log updated request headers/body info to access logs if ngx.req.get_headers()["x-debug-mode"] == "true" then setRequestLogs() end - redis.close(red) + dataStore:close() return end end @@ -97,7 +92,7 @@ end -- @param resourceKeys list of resourceKeys to search through -- @param tenant tenantId -- @param path path to look for -function _M.findRedisKey(resourceKeys, tenant, path) +function _M.findResource(resourceKeys, tenant, path) -- Check for exact match local redisKey = utils.concatStrings({"resources:", tenant, ":", path}) local cfRedisKey @@ -185,12 +180,12 @@ end -- @param red redis client instance -- @param obj List of policies containing a type and value field. This function reads the type field and routes it appropriately. -- @param apiKey optional subscription api key -function parsePolicies(red, obj, apiKey) +function parsePolicies(dataStore, obj, apiKey) for k, v in pairs (obj) do if v.type == 'reqMapping' then mapping.processMap(v.value) elseif v.type == 'rateLimit' then - rateLimit.limit(red, v.value, apiKey) + rateLimit.limit(dataStore, v.value, apiKey) elseif v.type == 'backendRouting' then backendRouting.setDynamicRoute(v.value) end diff --git a/tests/scripts/lua/lib/redis.lua b/tests/scripts/lua/lib/redis.lua index 0765b10..9b45124 100644 --- a/tests/scripts/lua/lib/redis.lua +++ b/tests/scripts/lua/lib/redis.lua @@ -27,6 +27,8 @@ describe('Testing Redis module', function() before_each(function() _G.ngx = fakengx.new() red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) operations = { GET = { backendUrl = 'https://httpbin.org/get', @@ -36,6 +38,8 @@ describe('Testing Redis module', function() end) it('should look up an api by one of it\'s member resources', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local sampleResource = cjson.decode([[ { "apiId": "a12341234", @@ -49,9 +53,11 @@ describe('Testing Redis module', function() ]]) red:hset('resources:test:v1/test', 'resources', cjson.encode(sampleResource)) - assert.are.same('a12341234', redis.resourceToApi(red, 'resources:test:v1/test')) + assert.are.same('a12341234', dataStore:resourceToApi('resources:test:v1/test')) end) it('should generate resource object to store in redis', function() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) -- Resource object with no policies or security local apiId = 12345 local resourceObj = { @@ -59,7 +65,7 @@ describe('Testing Redis module', function() operations = operations } local expected = resourceObj - local generated = cjson.decode(redis.generateResourceObj(operations, apiId)) + local generated = cjson.decode(dataStore:generateResourceObj(operations, apiId)) assert.are.same(expected, generated) -- Resource object with policy added @@ -76,7 +82,7 @@ describe('Testing Redis module', function() ]] resourceObj.operations.GET.policies = cjson.decode(policyList) expected = resourceObj - generated = cjson.decode(redis.generateResourceObj(operations, apiId)) + generated = cjson.decode(dataStore:generateResourceObj(operations, apiId)) assert.are.same(expected, generated) -- Resource object with security added @@ -89,7 +95,7 @@ describe('Testing Redis module', function() ]] resourceObj.operations.GET.security = cjson.decode(securityObj) expected = resourceObj - generated = cjson.decode(redis.generateResourceObj(operations, apiId)) + generated = cjson.decode(dataStore:generateResourceObj(operations, apiId)) assert.are.same(expected, generated) -- Resource object with multiple operations @@ -99,7 +105,7 @@ describe('Testing Redis module', function() security = {} } expected = resourceObj - generated = cjson.decode(redis.generateResourceObj(operations, apiId)) + generated = cjson.decode(dataStore:generateResourceObj(operations, apiId)) assert.are.same(expected, generated) local tenantObj = [[ @@ -114,30 +120,35 @@ describe('Testing Redis module', function() resourceObj.tenantNamespace = tenantObj.namespace resourceObj.tenantInstance = tenantObj.instance expected = resourceObj - generated = cjson.decode(redis.generateResourceObj(operations, apiId, tenantObj)) + generated = cjson.decode(dataStore:generateResourceObj(operations, apiId, tenantObj)) assert.are.same(expected, generated) end) it('should get a resource from redis', function() local key = 'resources:guest:hello' local field = 'resources' + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) -- resource doesn't exist in redis - local generated = redis.getResource(red, key, field) + local generated = dataStore:getResource(key, field) assert.are.same(nil, generated) -- resource exists in redis - local expected = redis.generateResourceObj(operations, nil) + local expected = dataStore:generateResourceObj(operations, nil) red:hset(key, field, expected) - generated = redis.getResource(red, key, field) + local dataStore = ds.initWithDriver(red) + generated = dataStore:getResource(key, field) assert.are.same(expected, generated) end) it('should create a resource in redis', function() local key = 'resources:guest:hello' local field = 'resources' - local expected = redis.generateResourceObj(operations, nil) - redis.createResource(red, key, field, expected) - local generated = redis.getResource(red, key, field) + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) + local expected = dataStore:generateResourceObj(operations, nil) + dataStore:createResource(key, field, expected) + local generated = dataStore:getResource(key, field) assert.are.same(expected, generated) end) @@ -145,31 +156,37 @@ describe('Testing Redis module', function() -- Key doesn't exist - throw 404 local key = 'resources:guest:hello' local field = 'resources' - redis.deleteResource(red, key, field) + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) + dataStore:deleteResource(key, field) assert.are.equal(ngx._exit, 404) -- Key exists - deleted properly local resourceObj = redis.generateResourceObj(operations, nil) - redis.createResource(red, key, field, resourceObj) + dataStore:createResource(key, field, resourceObj) local expected = 1 - local generated = redis.deleteResource(red, key, field) + local generated = dataStore:deleteResource(key, field) assert.are.same(expected, generated) end) it('shoud create an API Key subscription', function() local key = 'subscriptions:test:apikey' - redis.createSubscription(red, key) + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) + dataStore:createSubscription(key) assert.are.same(1, red:exists(key)) end) it('should delete an API Key subscription', function() -- API key doesn't exist in redis - throw 404 local key = 'subscriptions:test:apikey' - redis.deleteSubscription(red, key) + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) + dataStore:deleteSubscription(key) assert.are.equal(404, ngx._exit) -- API key to delete exists in redis red:set(key, '') - redis.deleteSubscription(red, key) + dataStore:deleteSubscription(key) assert.are.equal(0, red:exists(key)) end) diff --git a/tests/scripts/lua/routing.lua b/tests/scripts/lua/routing.lua index 529063f..8d5b68b 100644 --- a/tests/scripts/lua/routing.lua +++ b/tests/scripts/lua/routing.lua @@ -46,7 +46,7 @@ describe('Testing routing module', function() local expected = 'resources:guest:bp1/test/hello' local tenant = 'guest' local path = 'bp1/test/hello' - local actual = routing.findRedisKey(keys, tenant, path) + local actual = routing.findResource(keys, tenant, path) assert.are.same(expected, actual) expected = 'bp1/test/hello' actual = ngx.var.gatewayPath @@ -57,7 +57,7 @@ describe('Testing routing module', function() local expected = nil local tenant = 'guest' local path = 'bp1/bad/path' - local actual = routing.findRedisKey(keys, tenant, path) + local actual = routing.findResource(keys, tenant, path) assert.are.same(expected, actual) end) @@ -65,7 +65,7 @@ describe('Testing routing module', function() local expected = 'resources:guest:nobp' local tenant = 'guest' local path = 'nobp' - local actual = routing.findRedisKey(keys, tenant, path) + local actual = routing.findResource(keys, tenant, path) assert.are.same(expected, actual) expected = 'nobp' actual = ngx.var.gatewayPath @@ -76,7 +76,7 @@ describe('Testing routing module', function() local expected = 'resources:guest:noresource/' local tenant = 'guest' local path = 'noresource/' - local actual = routing.findRedisKey(keys, tenant, path) + local actual = routing.findResource(keys, tenant, path) assert.are.same(expected, actual) expected = 'noresource/' actual = ngx.var.gatewayPath diff --git a/tests/scripts/lua/security.lua b/tests/scripts/lua/security.lua index 5fd981e..0e04984 100644 --- a/tests/scripts/lua/security.lua +++ b/tests/scripts/lua/security.lua @@ -29,6 +29,8 @@ describe('API Key module', function() it('Checks an apiKey correctly', function() local red = fakeredis.new() local ngx = fakengx.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local ngxattrs = cjson.decode([[ { "tenant":"abcd", @@ -46,11 +48,13 @@ describe('API Key module', function() ]]) red:hset('resources:abcd:v1/test', 'resources', '{"apiId":"bnez"}') red:set('subscriptions:tenant:abcd:api:bnez:key:a1234', 'true') - local key = apikey.processWithRedis(red, securityObj, function() return "fakehash" end) + local key = apikey.process(dataStore, securityObj, function() return "fakehash" end) assert.same(key, 'a1234') end) it('Returns nil with a bad apikey', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local ngx = fakengx.new() local ngxattrs = cjson.decode([[ { @@ -68,11 +72,13 @@ describe('API Key module', function() } ]]) red:hset('resources:abcd:v1/test', 'resources', '{"apiId":"bnez"}') - local key = apikey.processWithRedis(red, securityObj, function() return "fakehash" end) + local key = apikey.process(dataStore, securityObj, function() return "fakehash" end) assert.falsy(key) end) it('Checks for a key with a custom header', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local ngx = fakengx.new() local ngxattrs = cjson.decode([[ { @@ -92,11 +98,13 @@ describe('API Key module', function() ]]) red:hset('resources:abcd:v1/test', 'resources', '{"apiId":"bnez"}') red:set('subscriptions:tenant:abcd:api:bnez:key:a1234', 'true') - local key = apikey.processWithRedis(red, securityObj, function() return "fakehash" end) + local key = apikey.process(dataStore, securityObj, function() return "fakehash" end) assert.same(key, 'a1234') end) it('Checks for a key with a custom header and hash configuration', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local ngx = fakengx.new() local ngxattrs = cjson.decode([[ { @@ -117,7 +125,7 @@ describe('API Key module', function() ]]) red:hset('resources:abcd:v1/test', 'resources', '{"apiId":"bnez"}') red:set('subscriptions:tenant:abcd:api:bnez:key:fakehash', 'true') - local key = apikey.processWithRedis(red, securityObj, function() return "fakehash" end) + local key = apikey.processWithHashFunction(dataStore, securityObj, function() return "fakehash" end) assert.same(key, 'fakehash') end) end) @@ -142,7 +150,7 @@ describe('OAuth security module', function() "scope":"resource" } ]] - local result = oauth.processWithRedis(red, cjson.decode(securityObj)) + local result = oauth.process(red, cjson.decode(securityObj)) assert.same(red:exists('oauth:providers:mock:tokens:test'), 1) assert(result) end) @@ -166,12 +174,14 @@ describe('OAuth security module', function() "scope":"resource" } ]] - local result = oauth.processWithRedis(red, cjson.decode(securityObj)) + local result = oauth.process(red, cjson.decode(securityObj)) assert.same(red:exists('oauth:providers:mock:tokens:bad'), 0) assert.falsy(result) end) it('Loads a facebook token from the cache without a valid app id', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local token = "test" local ngxattrs = [[ { @@ -192,11 +202,13 @@ describe('OAuth security module', function() } ]] red:set('oauth:providers:facebook:tokens:test', '{ "token":"good"}') - local result = oauth.processWithRedis(red, cjson.decode(securityObj)) + local result = oauth.process(dataStore, cjson.decode(securityObj)) assert.truthy(result) end) it('Loads a facebook token from the cache with a valid app id', function() local red = fakeredis.new() + local ds = require "lib/dataStore" + local dataStore = ds.initWithDriver(red) local token = "test" local appid = "app" local ngxattrs = [[ @@ -218,7 +230,7 @@ describe('OAuth security module', function() } ]] red:set('oauth:providers:facebook:tokens:testapp', '{"token":"good"}') - local result = oauth.processWithRedis(red, cjson.decode(securityObj)) + local result = oauth.process(dataStore, cjson.decode(securityObj)) assert.truthy(result) end) end) -- To stop receiving notification emails like this one, please contact ['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].