Add HiveOS Telegraf configuration

Signed-off-by: Julien Riou <julien@riou.xyz>

.gitignore

@@ -1,3 +1,6 @@
 docker/ssl/*
 !docker/ssl/.gitkeep
 docker/environment
+telegraf/hiveos.conf
+telegraf/hiveos/hiveos.conf
+telegraf/hiveos/venv

README.md

@@ -20,13 +20,15 @@ This stack is also known as the **TIG** stack.
 This guide uses [Docker](https://www.docker.com/). Ensure you have `docker`, `docker-compose` and `openssl` binaries
 installed.
 
-Write your miner address, grafana and influxdb credentials:
+Write grafana and influxdb credentials:
 
 ```
 cp -p docker/environment.example docker/environment
 vi docker/environment
 ```
 
+Depending on inputs, you should also set the miner address and HiveOS token.
+
 Generate a self-signed certificate:
 
 ```

docker/environment.example

@@ -1,4 +1,5 @@
 MINER_ADDRESS=
+HIVEOS_TOKEN=
 GF_SECURITY_ADMIN_USER=admin
 GF_SECURITY_ADMIN_PASSWORD=
 GF_USERS_DEFAULT_THEME=light

telegraf/README.md

@@ -1,3 +1,37 @@
 # Telegraf inputs
 
 Ensure to replace `MINER_ADDRESS` with your Ethereum wallet address.
+
+## HiveOS API
+
+### Create token
+
+On your [account](https://the.hiveos.farm/account) page, go to *Authentication Tokens*, search for *Personal Tokens* and
+create one for `telegraf`:
+
+![hiveos personal tokens](../images/hiveos-001.png)
+
+Enter your 2FA code if needed:
+
+![hiveos two factor auth](../images/hiveos-002.png)
+
+Clic on *Show*:
+
+![hiveos token show](../images/hiveos-003.png)
+
+And add this value to the `HIVEOS_TOKEN` environment variable (see *Quickstart* section in the [README](../README.md)).
+
+### Create Telegraf configuration
+
+```
+cd hiveos
+sudo apt install python3-virtualenv
+virtualenv venv
+source venv/bin/activate
+pip install -r requirements.txt
+read -s HIVEOS_TOKEN
+export HIVEOS_TOKEN
+python3 generate.py --verbose
+unset HIVEOS_TOKEN
+mv hiveos.conf ../
+```

telegraf/hiveos/farm.conf.j2

@@ -0,0 +1,8 @@
+[[inputs.http]]
+  name_override = "hiveos_farm"
+  urls = ["https://api2.hiveos.farm/api/v2/farms/{{farm_id}}"]
+  headers = {"Content-Type" = "application/json"}
+  headers = {"Authorization" = "Bearer ${HIVEOS_TOKEN}"}
+  data_format = "json"
+  tagexclude = ["url"]
+  tag_keys = ["name", "id"]

telegraf/hiveos/generate.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+import requests
+import os
+import sys
+import logging
+import argparse
+
+from jinja2 import Environment, FileSystemLoader
+
+logger = logging.getLogger(__name__)
+
+RETURN_OK=0
+RETURN_ERROR=1
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO,
+                        help='print more output')
+    parser.add_argument('-d', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG,
+                        default=logging.WARNING, help='print even more output')
+    args = parser.parse_args()
+    return args
+
+
+def setup_logging(args):
+    log_format = '%(levelname)s: %(message)s'
+    logging.basicConfig(format=log_format, level=args.loglevel)
+
+
+class HiveAPI:
+    def __init__(self, token):
+        session = requests.Session()
+        session.headers.update({'Content-Type': 'application/json', 'Authorization': f'Bearer {token}'})
+        self.session = session
+        self.base_url = 'https://api2.hiveos.farm/api/v2'
+        self.farms = {}
+        for farm_id in self.fetch_farms():
+            self.farms[farm_id] = self.fetch_workers(farm_id=farm_id)
+
+    def fetch_farms(self):
+        r = self.session.get(f'{self.base_url}/farms')
+        r.raise_for_status()
+        farms = r.json().get('data')
+        if farms:
+            return [f.get('id') for f in farms]
+
+    def fetch_workers(self, farm_id):
+        r = self.session.get(f'{self.base_url}/farms/{farm_id}/workers')
+        r.raise_for_status()
+        workers = r.json().get('data')
+        if workers:
+            return [w.get('id') for w in workers]
+
+
+def create_farm_configuration(farm_id):
+    return create_configuration(template_name='farm.conf.j2', template_variables={'farm_id': farm_id})
+
+
+def create_worker_configuration(farm_id, worker_id):
+    return create_configuration(template_name='worker.conf.j2',
+                                template_variables={'farm_id': farm_id, 'worker_id': worker_id})
+
+
+def create_configuration(template_name, template_variables):
+    template_path = os.path.split(os.path.abspath(__file__))[0]
+    loader = FileSystemLoader(template_path)
+    env = Environment(loader=loader)
+    template = env.get_template(template_name)
+    text = template.render(**template_variables)
+    return text
+
+
+def main():
+    args = parse_arguments()
+    setup_logging(args)
+
+    token = os.getenv('HIVEOS_TOKEN')
+    if not token:
+        logger.error('Missing HIVEOS_TOKEN environment variable')
+        return RETURN_ERROR
+
+    logger.info('crawling HiveOS API')
+    api = HiveAPI(token=token)
+
+    configurations = []
+    for farm_id in api.farms:
+        logger.info(f'generating configuration for farm {farm_id}')
+        configurations.append(create_farm_configuration(farm_id=farm_id))
+        for worker_id in api.farms[farm_id]:
+            logger.info(f'generating configuration for worker {worker_id}')
+            configurations.append(create_worker_configuration(farm_id=farm_id, worker_id=worker_id))
+
+    logger.info('writing configuration to file')
+    with open('hiveos.conf', 'w') as fd:
+        fd.write('\n'.join(configurations))
+
+    return RETURN_OK
+
+
+if __name__ == '__main__':
+    sys.exit(main())

telegraf/hiveos/hiveos.conf

@@ -0,0 +1,20 @@
+[[inputs.http]]
+  name_override = "hiveos_farm"
+  urls = ["https://api2.hiveos.farm/api/v2/farms/418240"]
+  headers = {"Content-Type" = "application/json"}
+  headers = {"Authorization" = "Bearer ${HIVEOS_TOKEN}"}
+  data_format = "json"
+  tagexclude = ["url"]
+  tag_keys = ["name", "id"]
+[[inputs.http]]
+  name_override = "hiveos_gpu"
+  urls = ["https://api2.hiveos.farm/api/v2/farms/418240/workers/1150566"]
+  headers = {"Content-Type" = "application/json"}
+  headers = {"Authorization" = "Bearer ${HIVEOS_TOKEN}"}
+  data_format = "json"
+  tagexclude = ["url"]
+  tag_keys = ["bus_number"]
+  json_query = "gpu_stats"
+  [inputs.http.tags]
+    farm_id = "418240"
+    worker_id = "1150566"

telegraf/hiveos/requirements.txt

@@ -0,0 +1,25 @@
+appdirs==1.4.3
+CacheControl==0.12.6
+certifi==2019.11.28
+chardet==3.0.4
+colorama==0.4.3
+contextlib2==0.6.0
+distlib==0.3.0
+distro==1.4.0
+html5lib==1.0.1
+idna==2.8
+ipaddr==2.2.0
+Jinja2==2.11.3
+lockfile==0.12.2
+MarkupSafe==1.1.1
+msgpack==0.6.2
+packaging==20.3
+pep517==0.8.2
+progress==1.5
+pyparsing==2.4.6
+pytoml==0.1.21
+requests==2.22.0
+retrying==1.3.3
+six==1.14.0
+urllib3==1.25.8
+webencodings==0.5.1

telegraf/hiveos/worker.conf.j2

@@ -0,0 +1,13 @@
+[[inputs.http]]
+  name_override = "hiveos_gpu"
+  urls = ["https://api2.hiveos.farm/api/v2/farms/{{farm_id}}/workers/{{worker_id}}"]
+  headers = {"Content-Type" = "application/json"}
+  headers = {"Authorization" = "Bearer ${HIVEOS_TOKEN}"}
+  data_format = "json"
+  tagexclude = ["url"]
+  tag_keys = ["bus_number"]
+  json_query = "gpu_stats"
+  [inputs.http.tags]
+    farm_id = "{{farm_id}}"
+    worker_id = "{{worker_id}}"
+