justin/Zerto_Exporter
1
0
Fork 0
mirror of https://github.com/recklessop/Zerto_Exporter.git synced 2026-07-07 01:33:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28 from recklessop/zvma-10-dev

Browse Source 
Zvma 10 dev
This commit is contained in:
Justin Paul
2024-05-30 21:07:55 -04:00
committed by GitHub
parent cdb9849dac ee10bb9a6b
commit 4e24f9b1f8
20 changed files with 2049 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -1,6 +1,7 @@
logs/*
.env/*
.DS_Store
uuid.txt
app/metrics
app/metrics.txt
app/statsmetrics
@@ -10,4 +11,8 @@ app/threads.txt
app/vrametrics
app/vrametrics.txt
app/__pycache__/*
app/microsoft.gpg
app/logs/*
app/zvma10/__pycache__/*
app/zvma9_7/__pycache__/*
app/temp.sh
Dockerfile
+31 -2
View File
@@ -1,9 +1,38 @@
FROM python:3.13.0b1-slim
FROM python:3.12.3-slim
EXPOSE 9999
# Install system dependencies
RUN apt-get update \
&& apt-get install -y \
curl \
gcc \
libffi-dev \
libssl-dev \
python3-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install Rust and Cargo using curl with IPv4 only
RUN CURL_IPRESOLVE=4 curl https://sh.rustup.rs -sSf | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
WORKDIR /usr/src/app
COPY app/* /usr/src/app/
# Set PYTHONPATH to include /usr/src/app
ENV PYTHONPATH=/usr/src/app
# Copy the zerto exporter into the container
COPY app /usr/src/app/
# Delete uuid.txt file if it exists
RUN [ -f uuid.txt ] && rm uuid.txt || echo "No uuid.txt file to delete"
# Install Python dependencies
# Set environment variable for PyO3 compatibility
ENV PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
RUN pip install --upgrade pip
RUN pip install --no-cache-dir -r requirements.txt
# Entry point for the container
CMD ["python", "python-node-exporter.py"]
app/.pypirc
+3
View File
@@ -0,0 +1,3 @@
[pypi]
username = __token__
password = pypi-AgEIcHlwaS5vcmcCJDZlMmZlNTc4LTI2NTgtNDVlZS04MDA2LTVmMGUzNTQyMzFmZAACKlszLCJjNTQzODQ3Yy1iOGQ1LTQwZDAtOTU5Yy0zNWIxNGVmM2NhNjkiXQAABiBdsEOQWVnk-qd-erTO48YJLKxiztWySeNQ45V6y45fkw
app/python-node-exporter.py
+244 -338
View File
@@ -4,179 +4,82 @@ import socketserver
import os
import ssl
import logging
from logging.handlers import RotatingFileHandler
import threading
import socket
from pyVim.connect import SmartConnect, Disconnect
from pyVmomi import vim
from time import sleep
from logging.handlers import RotatingFileHandler
from time import sleep, time
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.structures import CaseInsensitiveDict
from tinydb import TinyDB, Query
from tinydbstorage.storage import MemoryStorage
from tinydb.storages import MemoryStorage
from version import VERSION
from vmware.vcenter import vcsite
from zvma10.zvma import zvmsite
from posthog import Posthog
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
global start_time
start_time = time()
"""
Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually
"""
listen_port = int(os.getenv('LISTEN_PORT', 9999))
callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f')
verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't')
zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60')
zvm_port = os.environ.get('ZVM_PORT', '443')
client_id = os.environ.get('CLIENT_ID', 'api-script')
client_secret = os.environ.get('CLIENT_SECRET', 'js51tDM8oappYUGRJBhF7bcsedNoHA5j')
zvm_username = os.environ.get('ZVM_USERNAME', 'admin')
zvm_password = os.environ.get('ZVM_PASSWORD', 'Zertodata987!')
client_id = os.environ.get('CLIENT_ID', 'zerto-client')
client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk')
scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30))
api_timeout = int(os.environ.get('API_TIMEOUT', 5))
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
LOGLEVEL = os.environ.get('LOGLEVEL', 'DEBUG').upper()
DISABLE_STATS = os.environ.get('DISABLE_STATS', 'FALSE').upper()
version = str(VERSION)
vcenter_host = os.environ.get('VCENTER_HOST', 'vcenter.local')
vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50')
vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local')
vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'supersecret')
# Get the hostname of the machine
container_id = str(socket.gethostname())
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
log = logging.getLogger("Node-Exporter")
log.setLevel(LOGLEVEL)
log.addHandler(log_handler)
log.info(f"Zerto-Node-Exporter - Version {version}")
log.info(f"Log Level: {LOGLEVEL}")
log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret)
# Global Variables
token = ""
siteId = "NotSet"
siteName = "NotSet"
lastStats = CaseInsensitiveDict()
# Check if vCenter is set, if not disable VRA metrics
is_vcenter_set = True
if vcenter_host == "vcenter.local":
log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
is_vcenter_set = False
log.debug("vCenter data collection is enabled")
# Authentication Thread which handles authentication and token refresh for ZVM API
def ZvmAuthHandler():
log.debug("ZVMAuthHandler Thread Started")
expiresIn = 0
global token
global siteId
global siteName
retries = 0
while True:
if expiresIn < 30:
h = CaseInsensitiveDict()
h["Content-Type"] = "application/x-www-form-urlencoded"
d = CaseInsensitiveDict()
d["client_id"] = client_id
d["client_secret"] = client_secret
d["grant_type"] = "client_credentials"
uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token"
delay = 0
try:
response = requests.post(url=uri, data=d, headers=h, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
if 'access_token' not in responseJSON or 'expires_in' not in responseJSON:
log.error("Authentication response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
token = str(responseJSON.get('access_token'))
expiresIn = int(responseJSON.get('expires_in'))
log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds")
if response.status_code != 200:
log.error("Authentication request failed with status code " + str(response.status_code))
delay = 2 ** retries
sleep(delay)
retries += 1
continue
# Get Site ID and Name
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite"
delay = 0
try:
log.debug("Getting Site ID and Name")
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
log.debug(responseJSON)
if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON:
log.error("LocalSite API response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
else:
siteId = str(responseJSON.get('SiteIdentifier'))
siteName = str(responseJSON.get('SiteName'))
log.info("Site ID: " + siteId + " Site Name: " + siteName)
expiresIn -= 10 + delay
log.debug("Token Expires in " + str(expiresIn) + " seconds")
sleep(10)
vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!')
# Thread which gets VM level encryption statistics from ZVM API
def GetStatsFunc():
tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging
def GetStatsFunc(zvm_instance):
tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') #(storage=MemoryStorage) used for storing db on disk for debugging
dbvm = Query()
dbvpg = Query()
dbsite = Query()
zvm = zvm_instance
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.info("Got Auth Token!")
log.debug("token: " + str(token))
if (zvm.is_authenticated()):
log.debug("Stats Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
## Statistics API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/"
statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
statsapi_json = statsapi.json()
#log.debug(statsapi_json)
statsapi_json = None
statsapi_json = zvm.vms_statistics()
log.debug(statsapi_json)
vms_encryption_metrics = zvm.encryptiondetection_metrics_vms()
if statsapi_json is not None:
for vm in statsapi_json:
vmsiteinfo = zvm.vm(vmidentifier=vm['VmIdentifier'], vpgidentifier=vm['VpgIdentifier'])
if vmsiteinfo['ProtectedSite']['identifier'] == zvm.site_id:
log.debug(f"VM is protected at this site - {vm['VmIdentifier']}")
oldvmdata = dict()
# this part of the dictionary will never exist, so not sure why i need this as i set the key/values below in the vmem section.
if 'EncryptionMetrics' not in vm:
vm['EncryptionMetrics'] = {}
vm['VmName'] = None
vm['SiteId'] = zvm.site_id
CurrentIops = 0
CurrentWriteCounterInMBs = 0
@@ -186,16 +89,27 @@ def GetStatsFunc():
CurrentUnencryptedLBs = 0
CurrentTotalLBs = 0
CurrentPercentEncrypted = 0
CurrentTrendChangeLevel = 0
VMName = "NA"
oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
for vmem in vms_encryption_metrics:
if vmem['Link']['identifier'] == vm['VmIdentifier']:
log.debug(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}")
#print(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}")
vm['EncryptionMetrics']['EncryptedData'] = vmem['EncryptionMetrics']['EncryptedData']
vm['EncryptionMetrics']['NonEncryptedData'] = vmem['EncryptionMetrics']['NonEncryptedData']
vm['EncryptionMetrics']['TrendChangeLevel'] = vmem['EncryptionMetrics']['TrendChangeLevel']
vm['VmName'] = vmem['Link']['name']
log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
oldvmdata = tempdb.search((dbvm.VmIdentifier == vm['VmIdentifier']) & (dbvpg.VpgIdentifier == vm['VpgIdentifier']))
if (oldvmdata):
log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
log.debug(oldvmdata[0])
log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
log.debug("Old Data")
log.debug(oldvmdata)
log.debug(tempdb.update(vm, (dbvm.VmIdentifier == vm['VmIdentifier']) & (dbvpg.VpgIdentifier == vm['VpgIdentifier'])))
log.debug("New Data")
log.debug(vm)
log.debug("!@!@!@!@!@ Stats !@!@!@!@!@")
VMName = oldvmdata[0]['VmName']
log.debug("Current VM " + str(VMName))
@@ -205,10 +119,14 @@ def GetStatsFunc():
log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs'])
CurrentWriteCounterInMBs = abs(vm['WriteCounterInMBs'] - oldvmdata[0]['WriteCounterInMBs'])
log.debug("CurrentWriteCounterInMBs " + str(CurrentWriteCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionMetrics']['EncryptedData'] - oldvmdata[0]['EncryptionMetrics']['EncryptedData'])
log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs'])
CurrentUnencryptedLBs = abs(vm['EncryptionMetrics']['NonEncryptedData'] - oldvmdata[0]['EncryptionMetrics']['NonEncryptedData'])
log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
CurrentTrendChangeLevel = abs(vm['EncryptionMetrics']['TrendChangeLevel'] - oldvmdata[0]['EncryptionMetrics']['TrendChangeLevel'])
log.debug("CurrentTrendChangeLevel " + str(CurrentTrendChangeLevel))
CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
if CurrentTotalLBs != 0:
@@ -216,35 +134,28 @@ def GetStatsFunc():
else:
CurrentPercentEncrypted = 0
log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
else:
log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB")
log.info(f"{vm['VmIdentifier']} - {vm['VmName']} - No Record Found, Inserting into DB")
#insert original VM record to tempdb
log.debug(tempdb.insert(vm))
# update database with VM name, for easier display in Grafana Legends
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier']
try:
vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vapi_json = vapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
else:
log.debug("vapi_json: " + str(vapi_json))
tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier'])
log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName'])
VMName = vapi_json['VmName']
# Store Calculated Metrics
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
metricsDictionary["vm_TrendChangeLevel{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTrendChangeLevel
else:
log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}")
#print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}")
else:
log.debug("No VMS in Stats API")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('statsmetrics', 'w')
@@ -268,31 +179,23 @@ def GetStatsFunc():
sleep(1)
# Function which retrieves stats from various ZVM APIs and stores them in a metrics file
def GetDataFunc():
def GetDataFunc(zvm_instance):
tempdb = TinyDB(storage=MemoryStorage)
dbvm = Query()
zvm = zvm_instance
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.debug("Got Auth Token!")
log.debug("token: " + str(token))
if (zvm.is_authenticated()):
log.info("Data Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
### VPGs API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
vpg_json = service.json()
#log.debug(vpg_json)
vpg_json = None
vpg_json = zvm.vpgs()
if(vpg_json is not None):
log.debug("Got VPG JSON")
for vpg in vpg_json :
metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"]
metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"]
@@ -303,21 +206,25 @@ def GetDataFunc():
metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"]
metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"]
metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"]
if(vpg["FailSafeHistory"] is None):
metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = 0
metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = 0
else:
metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"]
metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"]
metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"]
metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"]
metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"]
else:
log.debug("No VPGs Found")
### Datastores APIs
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
ds_json = service.json()
#log.debug(ds_json)
ds_json = None
ds_json = zvm.datastores()
if(ds_json is not None):
log.debug("Got Datastores API")
for ds in ds_json :
log.debug(f"Processing {ds['DatastoreName']}")
metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"]
metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"]
metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"]
@@ -335,23 +242,16 @@ def GetDataFunc():
metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"]
else:
log.debug("No Datastores Found")
## VMs API
log.debug("Getting VMs API")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/"
vmapi_json = {}
try:
vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vmapi_json = vmapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
scratch_vols = None
scratch_vols = zvm.vms()
if(scratch_vols is not None):
log.debug("Got VMs API")
log.debug(vmapi_json)
for vm in vmapi_json :
for vm in scratch_vols:
log.debug("Processing VM: " + str(vm['VmName']))
log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId)
@@ -375,24 +275,18 @@ def GetDataFunc():
else:
log.debug("VM " + vm['VmIdentifier'] + " is protected to this site")
else:
log.debug("No VMs Found")
## Volumes API for Scratch Volumes
log.debug("Getting Scratch Volumes")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch"
volapi_json = {}
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
scratch_vols = None
scratch_vols = zvm.volumes(volumetype="scratch")
if(scratch_vols is not None):
log.debug("Got Scratch Volumes API")
if(bool(volapi_json)):
for volume in volapi_json :
for volume in scratch_vols:
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
@@ -403,24 +297,17 @@ def GetDataFunc():
percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100)
percentage_used = round(percentage_used, 1)
#metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used
else:
log.debug("No Scratch Volumes Found")
## Volumes API for Journal Volumes
log.debug("Getting Journal Volumes")
journal_vols = None
journal_vols = zvm.volumes(volumetype="journal")
volapi_json = {}
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal"
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got Journal Volumes API")
if(bool(volapi_json)):
if(journal_vols is not None):
log.debug("Journal Volumes Exist")
for volume in volapi_json :
for volume in journal_vols :
log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...")
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
@@ -441,6 +328,8 @@ def GetDataFunc():
metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1
else:
metricsDictionary[metrickey] = 1
else:
log.debug("No Journal Volumes Exist")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
log.debug("Writing metrics to file")
@@ -468,73 +357,31 @@ def GetDataFunc():
sleep(1)
# get VRA CPU and memory usage from vCenter Server
def GetVraMetrics():
# set up API endpoint and headers
log.debug("GetVraCpuMemory() called")
def GetVraMetrics(zvm_instance):
log.debug("GetVraMetrics thread started")
try:
metricsDictionary = {}
zvm = zvm_instance
while True:
vra_names = []
vras = []
global token
global siteId
global siteName
log.debug("Checking Token in VRA CPU MEM Collector")
if (token != ""):
log.debug("Auth Token Valid!")
log.debug("token: " + str(token))
if (zvm.is_authenticated()):
log.info("VRA CPU MEM Collector Running")
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
### VRA API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras"
vras_json = None
vras_json = zvm.vras()
log.debug(vras_json)
# make API call to get list of VRAs
try:
response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
except Exception as e:
log.error(f"Error connecting to {endpoint}: {e}")
return
else:
log.debug("Response from GET /v1/vras: %s", response.text)
# parse JSON response and get the name of each VRA
if is_vcenter_set:
# Disable SSL certificate verification
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.verify_mode = ssl.CERT_NONE
# connect to vCenter Server
si = None
try:
si = SmartConnect(host=vcenter_host, user=vcenter_user, pwd=vcenter_pwd, sslContext=context)
log.debug("Connected to vCenter Server %s", vcenter_host)
except Exception as e:
log.error(f"Error connecting to vCenter Server: {e}")
return
# get the root folder of the vCenter Server
content = si.RetrieveContent()
root_folder = content.rootFolder
# create a view for all VMs on the vCenter Server
view_manager = content.viewManager
vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True)
vras = response.json()
log.debug("VRA names: %s", vras)
if (vras_json is not None):
log.debug("VRA names: %s", vras_json)
log.debug(type(vras))
for vra in vras :
#vra_names.append(vra['VraName'])
for vra in vras_json :
# Gather other VRA Metrics from Zerto API into Metrics Diectionary
metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"]
metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"]
@@ -546,33 +393,29 @@ def GetVraMetrics():
metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"]
metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"]
log.debug("VRA Name: %s", vra['VraName'])
log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user}")
# get the CPU and memory usage for each VRA
if is_vcenter_set:
vm = None
for vm_obj in vm_view.view:
if vm_obj.name == vra['VraName']:
vm = vm_obj
break
if vm is not None:
log.debug("Found VRA VM in vCenter with name %s", vra['VraName'])
log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs")
try:
log.debug("Trying to get stats from vCenter module")
vradata = vc_connection.get_cpu_mem_used(vra['VraName'])
for item in vradata:
log.debug(item)
# get the CPU usage and memory usage for the VM
cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage
memory_usage_mb = vm.summary.quickStats.guestMemoryUsage
cpu_usage_mhz = vradata[0]
memory_usage_mb = vradata[1]
# print the CPU and memory usage for the VM
log.info(f"VM {vm.name} (name: {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz
metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb
except:
log.info(f"No VM found with name {vra['VraName']}, or unexpected response.")
else:
log.info(f"No VM found with name {vra['VraName']}")
# Disconnect from vCenter
Disconnect(si)
log.debug("No VRAs Found")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('vrametrics', 'w')
@@ -591,12 +434,13 @@ def GetVraMetrics():
txt_object.close()
# This function will get data every 10 seconds
log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds")
sleep(scrape_speed * 2)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
except Exception as e:
log.error(f"Error in GetVraMetrics: {e}")
# function which monitors the threads and restarts them if they die
def ThreadProbe():
@@ -605,31 +449,30 @@ def ThreadProbe():
log.debug("Thread Probe Started")
metricsDictionary = {}
log.debug("Is Auth Thread Alive")
if auth_thread.is_alive():
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is Data Thread Alive")
uptime = round((time() - start_time) / 60, 1)
metricsDictionary["exporter_uptime{ExporterInstance=\"" + container_id + "\"}"] = uptime
if data_thread.is_alive():
log.debug("Data Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("Data Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is Stats Thread Alive")
if stats_thread.is_alive():
log.debug("Stats Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("Stats Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is VRA Metrics Thread Alive")
if vra_metrics_thread.is_alive():
log.debug("VRA Metrics Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("VRA Metrics Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Writing Probe data to files")
log.debug("Writing Thread data to files")
file_object = open('threads', 'w')
txt_object = open('threads.txt', 'w')
for item in metricsDictionary :
@@ -642,40 +485,97 @@ def ThreadProbe():
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
log.debug("Trying to Close probe txt files")
log.debug("Trying to close Thread txt files")
file_object.close()
txt_object.close()
log.debug("Probe Thread Going to Sleep")
sleep(30)
#----------------run http server on port 9999-----------------
def WebServer():
log.info("Web Server Started")
PORT = 9999
#----------------run http server on port -----------------
def WebServer(port):
log.info(f"Web Server Starting on port {port}")
Handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
log.info(f"Webserver running on port {PORT}")
with socketserver.TCPServer(("", port), Handler) as httpd:
log.info(f"Webserver running on port {port}")
httpd.serve_forever()
def start_thread(target_func):
# start a new thread
log.debug(f"Starting thread for {target_func.__name__}")
thread = threading.Thread(target=target_func)
thread.daemon = True
thread.start()
# return the thread object
log.debug(f"Thread {target_func.__name__} started")
return thread
# start the threads
"""
Main Program Logic
"""
auth_thread = start_thread(ZvmAuthHandler)
data_thread = start_thread(GetDataFunc)
stats_thread = start_thread(GetStatsFunc)
vra_metrics_thread = start_thread(GetVraMetrics)
webserver_thread = start_thread(WebServer)
probe_thread = start_thread(ThreadProbe)
# Get the hostname of the machine
container_id = str(socket.gethostname())
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
log = logging.getLogger("Node-Exporter")
log.setLevel(LOGLEVEL)
log.addHandler(log_handler)
log.info(f"Zerto-Node-Exporter - Version {version}")
log.info(f"Log Level: {LOGLEVEL}")
log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret)
# Initialize zvmsite instance
zvm_instance = zvmsite(
host=zvm_url,
port=zvm_port,
username=zvm_username,
password=zvm_password,
client_id=client_id,
client_secret=client_secret,
loglevel=LOGLEVEL,
logger=log,
stats=DISABLE_STATS
)
# grant_type="client_credentials",
# Start the zvmsite authentication thread
zvm_instance.connect()
"""
Global Variables used by the program
"""
local_site_info = None
siteId = None
siteName = None
while(siteId is None):
if zvm_instance.is_authenticated():
sleep(2)
log.debug("Trying Set Global Vars")
siteId = zvm_instance.site_id
siteName = zvm_instance.site_name
lastStats = CaseInsensitiveDict()
# Check if vCenter is set, if not disable VRA metrics
is_vcenter_set = True
if vcenter_host == "vcenter.local":
log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
is_vcenter_set = False
log.debug("vCenter data collection is enabled")
vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug", logger=log)
# Starting threads
vra_metrics_thread = start_thread(lambda: GetVraMetrics(zvm_instance))
data_thread = start_thread(lambda: GetDataFunc(zvm_instance))
stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance))
log.debug("Starting VRA Metrics")
webserver_thread = start_thread(lambda: WebServer(listen_port))
probe_thread = start_thread(lambda: ThreadProbe)
log.debug(f"ThreadProbe just started on PID {probe_thread}")
# loop indefinitely
while True:
@@ -685,24 +585,30 @@ while True:
# restart the thread
log.error("Probe Thread Died - Restarting")
probe_thread = start_thread(ThreadProbe)
if not auth_thread.is_alive():
# restart the thread
log.error("Authentication Thread Died - Restarting")
auth_thread = start_thread(ZvmAuthHandler)
else:
print("Probe Thread is alive")
if not data_thread.is_alive():
# restart the thread
log.error("Data Thread Died - Restarting")
data_thread = start_thread(GetDataFunc)
data_thread = start_thread(GetDataFunc(zvm_instance))
else:
print("Data API Thread is alive")
if not stats_thread.is_alive():
# restart the thread
log.error("Stats Thread Died - Restarting")
stats_thread = start_thread(GetStatsFunc)
stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance))
else:
print("Stats API Thread is alive")
if not vra_metrics_thread.is_alive():
# restart the thread
log.error("VRA Metrics Thread Died - Restarting")
vra_metrics_thread = start_thread(GetVraMetrics)
vra_metrics_thread = start_thread(GetVraMetrics(zvm_instance))
else:
print("VRA Metrics Thread is alive")
if not webserver_thread.is_alive():
# restart the thread
log.error("Webserver Thread Died - Restarting")
webserver_thread = start_thread(WebServer)
webserver_thread = start_thread(WebServer(listen_port))
else:
print("WebServer Thread is alive")
sleep(api_timeout)
app/requirements.txt
+28 -5
View File
@@ -1,5 +1,28 @@
requests
tinydb
tinydb-storage
pyVmomi
pyVim
annotated-types==0.6.0
async-timeout==4.0.3
backoff==2.2.1
boto3==1.28.63
botocore==1.31.63
cachetools==5.3.1
certifi==2023.7.22
charset-normalizer==3.3.0
docopt==0.6.2
idna==3.4
jmespath==1.0.1
monotonic==1.6
posthog==3.0.2
prompt-toolkit==3.0.39
pydantic
pyflakes==3.1.0
Pygments==2.16.1
python-dateutil==2.8.2
pyvim==3.0.3
pyvmomi==8.0.2.0
redis==5.0.1
requests==2.32.0
s3transfer==0.7.0
six==1.16.0
tinydb==4.8.0
typing_extensions==4.8.0
urllib3==2.0.6
wcwidth==0.2.8
app/version.py
+1 -1
View File
@@ -1,5 +1,5 @@
# version.py
VERSION = "1.2.0"
VERSION = "2.1.0"
def main():
# Put your main program code here
app/vmware/__init__.py
+4
View File
@@ -0,0 +1,4 @@
print("Initializing vcenter package...")
#from .zvma import zvm
from .vcenter import vcsite
app/vmware/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
app/vmware/__pycache__/__init__.cpython-311.pyc
BIN
View File
Binary file not shown.
app/vmware/__pycache__/vcenter.cpython-310.pyc
BIN
View File
Binary file not shown.
app/vmware/__pycache__/vcenter.cpython-311.pyc
BIN
View File
Binary file not shown.
app/vmware/vcenter.py
+206
View File
@@ -0,0 +1,206 @@
# Class for holding variables related to a site.
from pyVim.connect import SmartConnect, Disconnect
from pyVmomi import vim, vmodl
import ssl
import datetime
import logging
import socket
from logging.handlers import RotatingFileHandler
class vcsite:
def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO", logger=None):
self.host = host
self.port = port
self.username = username
self.password = password
self.verify_ssl = verify_ssl
self.version = None
self.__conn__ = None
self.LOGLEVEL = loglevel.upper()
self.log = None
if logger is None:
#set log line format including container_id
container_id = str(socket.gethostname())
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
self.log = logging.getLogger("vCenter Module")
self.log.setLevel(self.LOGLEVEL)
self.log.addHandler(log_handler)
else:
self.log = logger
def connect(self):
self.log.info(f"Log Level set to {self.LOGLEVEL}")
if self.__conn__ is None:
context = ssl.create_default_context()
if not self.verify_ssl:
self.log.debug("dont verify SSL")
# Create an SSL context without certificate verification
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# connect to vCenter Server
si = None
try:
self.__conn__ = SmartConnect(host=self.host, user=self.username, pwd=self.password, sslContext=context)
about_info = self.__conn__.content.about
version = about_info.version
self.version = version
self.log.debug("Connected to vCenter Server %s", self.host)
except Exception as e:
self.log.error(f"Error connecting to vCenter Server: {e}")
def version(self):
return self.version
def get_cpu_mem_used(self, vra):
if vra == None:
self.log.debug("Get_cpu_mem_used called with no vm name...returning no data")
return
if self.__conn__ == None:
self.log.debug("Trying to get VRA stats without vCenter connection, trying to connect")
self.connect()
# get the root folder of the vCenter Server
try:
content = self.__conn__.RetrieveContent()
root_folder = content.rootFolder
except:
self.log.debug("Could not get content from vCenter when trying to get VRA stats")
# create a view for all VMs on the vCenter Server
view_manager = content.viewManager
vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True)
vm = None
for vm_obj in vm_view.view:
if str(vm_obj.name) == str(vra):
vm = vm_obj
if vm is not None:
self.log.debug(f"Found VRA VM in vCenter with name {vm.name}")
# get the CPU usage and memory usage for the VM
cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage
memory_usage_mb = vm.summary.quickStats.guestMemoryUsage
# print the CPU and memory usage for the VM
self.log.info(f"VM {vm.name} has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
return [cpu_usage_mhz, memory_usage_mb]
else:
self.log.debug(f"{vm_obj.name} is not a VRA")
raise ValueError("No VRA Found")
def get_write_iops(self, vm):
try:
content = self.__conn__.RetrieveContent()
except:
self.log.debug("Could not get content from vCenter when trying to get VRA stats")
# Find the virtual machine by name
vm_name = str(vm)
vm = None
for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view:
if obj.name == vm_name:
vm = obj
break
if vm is None:
print(f"Virtual machine '{vm_name}' not found")
return
# Get performance manager
perf_manager = content.perfManager
# Define the metric ID for write IOPS (counterId = 6)
metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="")
# calculate the last 60 seconds
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(seconds=60)
# Create a query specification for roll-up data
query_spec = vim.PerformanceManager.QuerySpec(
entity=vm,
metricId=[metric_id],
format="normal",
startTime=start_time,
endTime=end_time,
intervalId=20, # Use an appropriate interval for the roll-up data
)
# Query the performance statistics
result = perf_manager.QueryStats(querySpec=[query_spec])
if result:
# Get the average write IOPS for the last 60 seconds
average_write_iops = sum(result[0].value[0].value) / len(result[0].value[0].value)
print(f"Average write IOPS for the last 60 seconds for {vm_name}: {average_write_iops}")
return average_write_iops
else:
return None
def get_average_write_latency(self, vm):
try:
content = self.__conn__.RetrieveContent()
except:
self.log.debug("Could not get content from vCenter when trying to get VM stats")
# Find the virtual machine by name
vm_name = str(vm)
vm = None
for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view:
if obj.name == vm_name:
vm = obj
break
if vm is None:
self.log.debug(f"Virtual machine '{vm_name}' not found")
return None
# Get performance manager
perf_manager = content.perfManager
# Define the metric ID for write latency (counterId = X) - replace X with the correct counter ID
# You'll need to find the specific counter ID for write latency in your vSphere environment.
# The counter for write latency may vary based on your configuration.
metric_id = vim.PerformanceManager.MetricId(counterId=10, instance="") # Replace X with the correct counter ID
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(seconds=60)
# Create a query specification for roll-up data
query_spec = vim.PerformanceManager.QuerySpec(
entity=vm,
metricId=[metric_id],
format="normal",
startTime=start_time,
endTime=end_time,
intervalId=20, # Use an appropriate interval for the roll-up data
)
# Query the performance statistics
result = perf_manager.QueryStats(querySpec=[query_spec])
if result:
# Get the average write latency for the last 60 seconds
if result[0].value[0].value:
average_write_latency = sum(result[0].value[0].value) / len(result[0].value[0].value)
self.log.info(f"Average write latency for the last 60 seconds for {vm_name}: {average_write_latency}")
return average_write_latency
return None
def disconnect(self):
if self.__conn__ == None:
self.log.debug(f"vCenter disconnect requested, but not currently connected.")
return
# Disconnect from vCenter
Disconnect(self.__conn__)
self.__conn__ = None
self.version = None
self.log.debug(f"Disconnected from vCenter")
app/vmware/version.py
+10
View File
@@ -0,0 +1,10 @@
# version.py
VERSION = "0.1.0"
def main():
# Put your main program code here
print(VERSION)
if __name__ == '__main__':
# This code will be executed only when the file is run as the main program
main()
app/zvma10/__init__.py
+4
View File
@@ -0,0 +1,4 @@
print("Initializing zvma10 package...")
#from .zvma import zvm
from .zvma import zvmsite
app/zvma10/test.py
+26
View File
@@ -0,0 +1,26 @@
import requests
from requests import Request, Session
# Create a Request object
url = 'http://httpbin.org/post'
headers = {'Content-Type': 'application/json'}
data = {'key': 'value'}
req = Request('POST', url, data=data, headers=headers)
# Prepare the request
prepared_req = req.prepare()
# Print the prepared request details
print("Prepared Request:")
print(f"URL: {prepared_req.url}")
print(f"Method: {prepared_req.method}")
print(f"Headers: {prepared_req.headers}")
print(f"Body: {prepared_req.body}")
# Send the request using a Session
with Session() as s:
response = s.send(prepared_req)
# Print the response
print(f"\nResponse Status Code: {response.status_code}")
print(response.text)
app/zvma10/version.py
+10
View File
@@ -0,0 +1,10 @@
# version.py
VERSION = "0.1.0"
def main():
# Put your main program code here
print(VERSION)
if __name__ == '__main__':
# This code will be executed only when the file is run as the main program
main()
app/zvma10/zvma.py
+1202
View File
File diff suppressed because it is too large Load Diff
app/zvma9_7/GetStatsFunc.py
+126
View File
@@ -0,0 +1,126 @@
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.structures import CaseInsensitiveDict
from tinydb import TinyDB, Query
from tinydbstorage.storage import MemoryStorage
from logging.handlers import RotatingFileHandler
# Function to get VM Encryption Data from ZVMa version 9.7
def GetStatsFunc():
tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging
dbvm = Query()
dbvpg = Query()
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.info("Got Auth Token!")
log.debug("token: " + str(token))
log.debug("Stats Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
## Statistics API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/"
statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
statsapi_json = statsapi.json()
#log.debug(statsapi_json)
for vm in statsapi_json:
oldvmdata = dict()
CurrentIops = 0
CurrentWriteCounterInMBs = 0
CurrentSyncCounterInMBs = 0
CurrentNetworkTrafficCounterInMBs = 0
CurrentEncryptedLBs = 0
CurrentUnencryptedLBs = 0
CurrentTotalLBs = 0
CurrentPercentEncrypted = 0
VMName = "NA"
oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
if (oldvmdata):
log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
log.debug(oldvmdata[0])
log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
log.debug("!@!@!@!@!@ Stats !@!@!@!@!@")
VMName = oldvmdata[0]['VmName']
log.debug("Current VM " + str(VMName))
CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter'])
log.debug("CurrentIops " + str(CurrentIops))
CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs'])
log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs'])
log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs'])
log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
if CurrentTotalLBs != 0:
CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100)
else:
CurrentPercentEncrypted = 0
log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
else:
log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB")
#insert original VM record to tempdb
log.debug(tempdb.insert(vm))
# update database with VM name, for easier display in Grafana Legends
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier']
try:
vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vapi_json = vapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
else:
log.debug("vapi_json: " + str(vapi_json))
tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier'])
log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName'])
VMName = vapi_json['VmName']
# Store Calculated Metrics
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('statsmetrics', 'w')
txt_object = open('statsmetrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
sleep(scrape_speed)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
app/zvma9_7/__init__.py
+3
View File
@@ -0,0 +1,3 @@
print("Initializing zvma9_7 package...")
from .GetStatsFunc import GetStatsFunc
docker-compose.yml
+4 -5
View File
@@ -1,9 +1,8 @@
version: "3.3"
services:
custom-exporter-in-python:
zerto-exporter:
build: .
command: python python-node-exporter.py
ports:
- "9999:9999"
environment:
@@ -11,10 +10,10 @@ services:
- ZVM_HOST=192.168.50.60
- ZVM_PORT=443
- CLIENT_ID=api-script
- CLIENT_SECRET=js51tDM8oappYUGRJBhF7bcsedNoHA5j
- CLIENT_SECRET=fcYMFuA5TkIUwp6b3hDUxim0f32z8erk
- LOGLEVEL=INFO #Valid settings are CRITICAL, ERROR, WARNING, INFO, DEBUG
- VCENTER_HOST=192.168.50.50
- VCENTER_USER=administrator@vsphere.local
- VCENTER_PASSWORD=password
- VCENTER_PASSWORD=Zertodata987!
volumes:
- "./app:/usr/src/app:rw"
- "./logs:/usr/src/app/logs/"