GitHub-Mirror
/
ConsulWatcher
mirror of
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added support for multiple service instances

This commit is contained in:
Mark van Renswoude 2020-07-07 11:03:36 +02:00
parent 2520824fe6
commit 5134ea748c
9 changed files with 213 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
node_modules
config/*
!config/*.example.js
!config/*.example.js
!config/setindent.js

2
README.md
View File

@ -1,5 +1,5 @@
# ConsulWatcher
This is a much simplified version of HashiCorp's Consul-Template, because what it accomplishes is awesome but I really dislike the Go template syntax. All configuration in this version are plain JavaScript.
This is a much simplified version of HashiCorp's Consul-Template, because what it accomplishes is awesome but I really dislike the Go template syntax. All configuration in this version is plain JavaScript.
It's main purpose is creating output files based on a Consul catalog, much like Consul-Template, but since the update handlers are just JavaScript functions you are free to do whatever you want, like calling a webservice or using a template library.

108
config/index.example.js
View File

@ -1,48 +1,49 @@
const winston = require('winston');
const setIndent = require('./setindent');
const config = {
/*
Consul agent configuration
Determines the agent or server which will be queried and
monitored for the service catalog.
Recommended to be a local agent connected to the cluster.
Passed to the initialization of the Node Consul client.
For all options, see: https://github.com/silas/node-consul#init
*/
consul: {
host: 'pc-mvrenswoude'
},
/*
Logging
See: https://github.com/winstonjs/winston#transports
*/
logging: {
transports: [
new winston.transports.Console({
level: 'debug',
timestamp: true,
format: winston.format.combine(
winston.format.colorize(),
winston.format.simple()
)
})
]
},
onUpdate: [],
afterUpdate: null
};
/*
Consul agent configuration
Determines the agent or server which will be queried and
monitored for the service catalog.
Recommended to be a local agent connected to the cluster.
Passed to the initialization of the Node Consul client.
For all options, see: https://github.com/silas/node-consul#init
*/
config.consul = {
host: 'localhost'
}
/*
Logging
See: https://github.com/winstonjs/winston#transports
*/
const winston = require('winston');
config.logging = {
transports: [
new winston.transports.Console({
level: 'debug',
timestamp: true,
format: winston.format.combine(
winston.format.colorize(),
winston.format.simple()
)
})
]
};
/*
@ -65,26 +66,41 @@ config.logging = {
const fs = require('fs').promises;
config.onUpdate.push((catalog, logger) =>
config.onUpdate.push(async (catalog, logger) =>
{
// Use catalog parameter to generate output
let output = '';
for (const service of catalog.services)
{
output +=
`Service: ${service.name}
Tags: ${JSON.stringify(service.tags)}
Address: ${await service.getAddress()}
Port: ${await service.getPort()}
let instances = '';
`;
for (const instance of await service.getInstances())
{
instances += setIndent(2, `
Address: ${instance.address}
Port: ${instance.port}
`);
}
output += setIndent(`
Service: ${service.name}
Tags: ${JSON.stringify(service.tags)}
`);
output += instances + '\n';
};
await fs.writeFile('example-output.txt', output);
});
// Example on how to split the update handlers into a separate file
config.onUpdate.push(require('./included.example'));
/*
afterUpdate handler
This is a single handler which is called after all the onUpdate handlers

49
config/setindent.js Normal file
View File

@ -0,0 +1,49 @@
// Inspired by dedent: https://github.com/dmnd/dedent
function setIndent(indentOrValue, value = null)
{
let indent;
if (typeof indentOrValue === 'number')
indent = indentOrValue;
else
{
indent = 0;
value = indentOrValue;
}
const lines = value.split('\n');
let minIndent = null;
// Determine minimum indent present in the value
for (const line of lines)
{
const whitespace = line.match(/^(\s+)/);
if (whitespace !== null)
minIndent = minIndent === null ? whitespace[1].length : Math.min(whitespace[1].length, minIndent);
}
// Calculate the difference to the requested indentation
const indentDelta = minIndent === null ? indent : minIndent - indent;
let result;
if (indentDelta < 0)
{
const add = ' '.repeat(-indentDelta);
result = lines.map(line => add + line).join('\n');
}
else
result = lines.map(line => line.slice(indentDelta)).join('\n');
// If the first or last line is empty, trim it (allows the template string to
// start at the next line to align properly). Similarly, if the last line is
// only whitespace, keep the newline but trim the spaces.
return result.replace(/^\n/g, '').replace(/[ \t]+$/g, '');
}
module.exports = setIndent;

13
index.js
View File

@ -1,9 +1,14 @@
const options = require('minimist')(process.argv.slice(2));
const winston = require('winston');
const ConsulCatalog = require('./lib/consulcatalog');
const logger = require('./lib/logger');
const config = require('./config');
const config = options.hasOwnProperty('config') ? require(options.config) : require('./config');
const logger = winston.createLogger({
transports: config.logging.transports,
});
const catalog = new ConsulCatalog(logger, config);
// TODO detect if the connection is down for too long, allow a custom notification to be sent
// TODO provide a way to easily switch between configs, for multiple environments

154
lib/consulcatalog.js
View File

@ -68,7 +68,7 @@ class ConsulCatalog
serviceByName(name)
{
return this._services.find(service => service.name == name) || null;
return this._services.find(service => service.name === name) || null;
}
@ -97,7 +97,7 @@ class ConsulCatalog
self._services = self._services.filter(service =>
{
const serviceIndex = serviceNames.indexOf(service.name);
if (serviceIndex == -1)
if (serviceIndex === -1)
{
// Previously detected service no longer appears in Consul, remove
// any watches that may be present and remove it from the list
@ -116,12 +116,12 @@ class ConsulCatalog
});
// All remaining entries in serviceNames are new
serviceNames.forEach(name =>
for (const name of serviceNames)
{
self._logger.debug(`Found new service: ${name}`)
self._services.push(new ConsulService(self, name, data[name]));
changed = true;
});
};
return changed;
}
@ -139,7 +139,7 @@ class ConsulCatalog
}
if (self._debouncedUpdate == null)
if (self._debouncedUpdate === null)
{
self._debouncedUpdate = debounce(() =>
{
@ -149,11 +149,11 @@ class ConsulCatalog
self._logger.info('Running update handlers');
const handlerPromises = [];
self._config.onUpdate.forEach(handler =>
for (const handler of self._config.onUpdate)
{
const handlerPromise = Promise.resolve(handler(self, self._logger));
handlerPromises.push(handlerPromise);
});
};
Promise.all(handlerPromises)
.then(() =>
@ -171,6 +171,11 @@ class ConsulCatalog
self._doUpdate();
}
});
})
.catch(e =>
{
// TODO better exception handling
self._logger.error('Error while running update handlers: ', e);
});
}, 500);
}
@ -189,69 +194,22 @@ class ConsulService
this._logger = catalog._logger;
this._watch = null;
this._rawData = null;
this._rawDataPromise = null;
this._instances = null;
this.name = name;
this.tags = tags;
}
async getAddress()
{
var rawData;
try
{
rawData = await this._getRawData();
}
catch(e)
{
// TODO better exception handling
this._logger.error('Error while retrieving service status', err);
return null;
}
if (rawData.length == 0)
return null;
if (rawData[0].Service.Address != '')
return rawData[0].Service.Address;
return rawData[0].Node.Address;
}
async getPort()
{
var rawData;
try
{
rawData = await this._getRawData();
}
catch(e)
{
// TODO better exception handling
this._logger.error('Error while retrieving service status', err);
return null;
}
return rawData.length > 0 ? rawData[0].Service.Port : null;
}
// TODO getHealth
_getRawData()
async getInstances()
{
const self = this;
if (self._rawDataPromise !== null)
return self._rawDataPromise;
if (self._instances !== null)
return Promise.resolve(self._instances);
// Get status information for the service and start watching it
self._rawDataPromise = new Promise((resolve, reject) =>
return new Promise((resolve, reject) =>
{
let firstResponse = true;
@ -269,7 +227,7 @@ class ConsulService
if (firstResponse)
{
firstResponse = false;
resolve(self._rawData);
resolve(self._instances);
}
});
@ -284,13 +242,8 @@ class ConsulService
firstResponse = false;
reject(err);
}
// Try again the next time
self._rawDataPromise = null;
});
});
return self._rawDataPromise;
}
@ -317,16 +270,77 @@ class ConsulService
_applyHealthData(data)
{
if (data == this._rawData)
return false;
const self = this;
const isUpdate = self._instances !== null;
let changed = false;
// Remove instances that no longer exist
const instanceIds = {};
for (const dataInstance of data)
instanceIds[dataInstance.Service.ID] = dataInstance;
self._instances = (self._instances != null ? self._instances : []).filter(instance =>
{
if (instanceIds.hasOwnProperty(instance.id))
{
// Previously detected instance no longer appears in Consul,
// remove it from the list
changed = true;
return false;
}
if (instance._applyHealthData(instanceIds[instance.id]))
changed = true;
// Remove from instanceIds to indicate it has already been applied
delete instanceIds[instance.id];
return true;
});
// All remaining entries in instanceIds are new
for (const id of Object.keys(instanceIds))
{
self._logger.debug(`Found new service instance: ${id}`)
self._instances.push(new ConsulServiceInstance(id, instanceIds[id]));
changed = true;
};
const isUpdate = this._rawData != null;
this._rawData = data;
// If this is the first time we've received data, it is guaranteed to be the result of
// an update handler requesting this data and the handlers do not need to be called again.
return isUpdate;
return isUpdate && changed;
}
}
class ConsulServiceInstance
{
constructor(id, data)
{
this.id = id;
this.tags = [];
this.address = null;
this.port = null;
this.rawData = null;
this._applyHealthData(data);
}
_applyHealthData(data)
{
this.rawData = data;
this.id = data.Service.ID;
this.tags = data.Service.Tags;
this.address = data.Service.Address !== '' ? data.Service.Address : data.Node.Address;
this.port = data.Service.Port;
}
}
module.exports = ConsulCatalog;

9
lib/logger.js
View File

@ -1,9 +0,0 @@
const winston = require('winston');
const config = require('../config');
// TODO make configurable
let logger = winston.createLogger({
transports: config.logging.transports,
});
module.exports = logger;

10
package-lock.json generated
View File

@ -82,11 +82,6 @@
"resolved": "https://registry.npmjs.org/debounce/-/debounce-1.2.0.tgz",
"integrity": "sha512-mYtLl1xfZLi1m4RtQYlZgJUNQjl4ZxVnHzIR8nLLgi4q1YT8o/WM+MK/f8yfcc9s5Ir5zRaPZyZU6xs1Syoocg=="
},
"dedent": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
"integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw="
},
"enabled": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz",
@ -144,6 +139,11 @@
"triple-beam": "^1.3.0"
}
},
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",

2
package.json
View File

@ -20,7 +20,7 @@
"dependencies": {
"consul": "^0.37.0",
"debounce": "^1.2.0",
"dedent": "^0.7.0",
"minimist": "^1.2.5",
"winston": "^3.3.3"
},
"devDependencies": {