finished exporter refactor

Ready for some beta testing with Zerto 10.0
This commit is contained in:
2023-12-21 10:34:41 -05:00
parent deae17aca9
commit 7618fa7d04
8 changed files with 877 additions and 1326 deletions
+1
View File
@@ -1,6 +1,7 @@
logs/*
.env/*
.DS_Store
uuid.txt
app/metrics
app/metrics.txt
app/statsmetrics
-719
View File
@@ -1,719 +0,0 @@
import requests
import http.server
import socketserver
import os
import ssl
import logging
from logging.handlers import RotatingFileHandler
import threading
import socket
from pyVim.connect import SmartConnect, Disconnect
from pyVmomi import vim
from time import sleep
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.structures import CaseInsensitiveDict
from tinydb import TinyDB, Query
from tinydbstorage.storage import MemoryStorage
from version import VERSION
from vmware.vcenter import vcsite
from zvma10.zvma import zvmsite
from posthog import Posthog
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
"""
Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually
"""
callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f')
verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't')
zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60')
zvm_port = os.environ.get('ZVM_PORT', '443')
client_id = os.environ.get('CLIENT_ID', 'api-script')
client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk')
scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30))
api_timeout = int(os.environ.get('API_TIMEOUT', 5))
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
version = str(VERSION)
vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50')
vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local')
vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!')
"""
Global Variables used by the program
"""
token = ""
siteId = "NotSet"
siteName = "NotSet"
siteZvmVersion = ""
siteVcVersion = ""
siteZvmMajorVersion = ""
siteZvmMinorVersion = ""
siteZvmUpdateVersion = ""
siteZvmPatchVersion = ""
lastStats = CaseInsensitiveDict()
# Authentication Thread which handles authentication and token refresh for ZVM API
def ZvmAuthHandler():
log.debug("ZVMAuthHandler Thread Started")
expiresIn = 0
global token
global siteId
global siteName
retries = 0
while True:
if expiresIn < 30:
h = CaseInsensitiveDict()
h["Content-Type"] = "application/x-www-form-urlencoded"
d = CaseInsensitiveDict()
d["client_id"] = client_id
d["client_secret"] = client_secret
d["grant_type"] = "client_credentials"
uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token"
delay = 0
try:
response = requests.post(url=uri, data=d, headers=h, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
if 'access_token' not in responseJSON or 'expires_in' not in responseJSON:
log.error("Authentication response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
token = str(responseJSON.get('access_token'))
expiresIn = int(responseJSON.get('expires_in'))
log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds")
if response.status_code != 200:
log.error("Authentication request failed with status code " + str(response.status_code))
delay = 2 ** retries
sleep(delay)
retries += 1
continue
# Get Site ID and Name
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite"
delay = 0
try:
log.debug("Getting Site ID and Name")
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
log.debug(responseJSON)
if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON:
log.error("LocalSite API response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
else:
siteId = str(responseJSON.get('SiteIdentifier'))
siteName = str(responseJSON.get('SiteName'))
siteZvmVersion = str(responseJSON.get('Version'))
siteVcVersion = str(responseJSON.get('SiteTypeVersion'))
# Break out ZVM version strings
siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".")
siteZvmUpdateVersion = siteZvmUpdateVersion[0]
if (len(siteZvmUpdateVersion) > 1):
siteZvmPatchVersion = siteZvmUpdateVersion[1]
else:
siteZvmPatchVersion = "0"
log.info("Site ID: " + siteId + " Site Name: " + siteName)
expiresIn -= 10 + delay
log.debug("Token Expires in " + str(expiresIn) + " seconds")
sleep(10)
'''
# Thread which gets VM level encryption statistics from ZVM API
def GetStatsFunc():
tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging
dbvm = Query()
dbvpg = Query()
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.info("Got Auth Token!")
log.debug("token: " + str(token))
log.debug("Stats Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
## Statistics API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/"
statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
statsapi_json = statsapi.json()
#log.debug(statsapi_json)
for vm in statsapi_json:
oldvmdata = dict()
CurrentIops = 0
CurrentWriteCounterInMBs = 0
CurrentSyncCounterInMBs = 0
CurrentNetworkTrafficCounterInMBs = 0
CurrentEncryptedLBs = 0
CurrentUnencryptedLBs = 0
CurrentTotalLBs = 0
CurrentPercentEncrypted = 0
VMName = "NA"
oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
if (oldvmdata):
log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
log.debug(oldvmdata[0])
log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
log.debug("!@!@!@!@!@ Stats !@!@!@!@!@")
VMName = oldvmdata[0]['VmName']
log.debug("Current VM " + str(VMName))
CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter'])
log.debug("CurrentIops " + str(CurrentIops))
CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs'])
log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs'])
log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs'])
log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
if CurrentTotalLBs != 0:
CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100)
else:
CurrentPercentEncrypted = 0
log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
else:
log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB")
#insert original VM record to tempdb
log.debug(tempdb.insert(vm))
# update database with VM name, for easier display in Grafana Legends
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier']
try:
vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vapi_json = vapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
else:
log.debug("vapi_json: " + str(vapi_json))
tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier'])
log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName'])
VMName = vapi_json['VmName']
# Store Calculated Metrics
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('statsmetrics', 'w')
txt_object = open('statsmetrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
sleep(scrape_speed)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
'''
# Function which retrieves stats from various ZVM APIs and stores them in a metrics file
def GetDataFunc():
tempdb = TinyDB(storage=MemoryStorage)
dbvm = Query()
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.debug("Got Auth Token!")
log.debug("token: " + str(token))
log.info("Data Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
### VPGs API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
vpg_json = service.json()
#log.debug(vpg_json)
for vpg in vpg_json :
metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"]
metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"]
metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"]
metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"]
metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"]
metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"]
metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"]
metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"]
metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"]
metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"]
metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"]
metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"]
metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"]
metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"]
### Datastores APIs
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
ds_json = service.json()
#log.debug(ds_json)
for ds in ds_json :
log.debug(f"Processing {ds['DatastoreName']}")
metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"]
metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"]
metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"]
metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"]
metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"]
metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"]
metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"]
## VMs API
log.debug("Getting VMs API")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/"
vmapi_json = {}
try:
vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vmapi_json = vmapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got VMs API")
log.debug(vmapi_json)
for vm in vmapi_json :
log.debug("Processing VM: " + str(vm['VmName']))
log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId)
if siteId == vm['ProtectedSite']['identifier']:
log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site")
if not isinstance(vm["ActualRPO"], int):
vm["ActualRPO"] = -1
metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"]
metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"]
metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"]
metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"]
metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"]
metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"]
metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"]
metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"]
metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"]
metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"]
metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"]
log.debug("Processed VM: " + str(vm['VmName']))
else:
log.debug("VM " + vm['VmIdentifier'] + " is protected to this site")
## Volumes API for Scratch Volumes
log.debug("Getting Scratch Volumes")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch"
volapi_json = {}
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got Scratch Volumes API")
if(bool(volapi_json)):
for volume in volapi_json :
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
if (metrickey in metricsDictionary):
metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"]
else:
metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"]
percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100)
percentage_used = round(percentage_used, 1)
#metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used
## Volumes API for Journal Volumes
log.debug("Getting Journal Volumes")
volapi_json = {}
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal"
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got Journal Volumes API")
if(bool(volapi_json)):
log.debug("Journal Volumes Exist")
for volume in volapi_json :
log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...")
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
metrickey = "vm_journal_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
if (metrickey in metricsDictionary):
metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"]
else:
metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"]
metrickey = "vm_journal_volume_provisioned_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
if (metrickey in metricsDictionary):
metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["ProvisionedInBytes"]
else:
metricsDictionary[metrickey] = volume["Size"]["ProvisionedInBytes"]
metrickey = "vm_journal_volume_count{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
if (metrickey in metricsDictionary):
metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1
else:
metricsDictionary[metrickey] = 1
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
log.debug("Writing metrics to file")
file_object = open('metrics', 'w')
txt_object = open('metrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
log.debug("Metrics written to file")
# This function will get data every 10 seconds
log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
sleep(scrape_speed)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
# get VRA CPU and memory usage from vCenter Server
def GetVraMetrics():
# set up API endpoint and headers
log.debug("GetVraCpuMemory() called")
metricsDictionary = {}
while True:
vra_names = []
vras = []
global token
global siteId
global siteName
log.debug("Checking Token in VRA CPU MEM Collector")
if (token != ""):
log.debug("Auth Token Valid!")
log.debug("token: " + str(token))
log.info("VRA CPU MEM Collector Running")
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
### VRA API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras"
# make API call to get list of VRAs
try:
response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
except Exception as e:
log.error(f"Error connecting to {uri}: {e}")
return
else:
log.debug("Response from GET /v1/vras: %s", response.text)
# parse JSON response and get the name of each VRA
vras = response.json()
log.debug("VRA names: %s", vras)
log.debug(type(vras))
for vra in vras :
# Gather other VRA Metrics from Zerto API into Metrics Diectionary
metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"]
metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"]
metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"]
metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"]
metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"]
metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"]
metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"]
metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"]
metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"]
log.debug("VRA Name: %s", vra['VraName'])
log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}")
# get the CPU and memory usage for each VRA
if is_vcenter_set:
log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs")
try:
log.debug("Trying to get stats from vc module")
vradata = vc_connection.get_cpu_mem_used(vra['VraName'])
# get the CPU usage and memory usage for the VM
cpu_usage_mhz = vradata[0]
memory_usage_mb = vradata[1]
# print the CPU and memory usage for the VM
log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz
metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb
except:
log.info(f"No VM found with name {vra['VraName']}")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('vrametrics', 'w')
txt_object = open('vrametrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
# This function will get data every 10 seconds
log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds")
sleep(scrape_speed * 2)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
# function which monitors the threads and restarts them if they die
def ThreadProbe():
global container_id
while True:
log.debug("Thread Probe Started")
metricsDictionary = {}
log.debug("Is Auth Thread Alive")
if auth_thread.is_alive():
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is Data Thread Alive")
if data_thread.is_alive():
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
#log.debug("Is Stats Thread Alive")
#if stats_thread.is_alive():
# metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
#else:
# metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is VRA Metrics Thread Alive")
if vra_metrics_thread.is_alive():
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Writing Probe data to files")
file_object = open('threads', 'w')
txt_object = open('threads.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
log.debug("Trying to Close probe txt files")
file_object.close()
txt_object.close()
log.debug("Probe Thread Going to Sleep")
sleep(30)
#----------------run http server on port 9999-----------------
def WebServer():
log.info("Web Server Started")
PORT = 9999
Handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
log.info(f"Webserver running on port {PORT}")
httpd.serve_forever()
def start_thread(target_func):
# start a new thread
thread = threading.Thread(target=target_func)
thread.start()
# return the thread object
return thread
"""
Main Program Logic
"""
# Initialize zvmsite instance
zvm_instance = zvmsite(
host=zvm_url,
port=zvm_port,
client_id=client_id,
client_secret=client_secret,
grant_type="client_credentials",
loglevel=LOGLEVEL
)
# Start the zvmsite authentication thread
zvm_instance.connect()
# Get the hostname of the machine
container_id = str(socket.gethostname())
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
log = logging.getLogger("Node-Exporter")
log.setLevel(LOGLEVEL)
log.addHandler(log_handler)
log.info(f"Zerto-Node-Exporter - Version {version}")
log.info(f"Log Level: {LOGLEVEL}")
log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret)
# Check if vCenter is set, if not disable VRA metrics
is_vcenter_set = True
if vcenter_host == "vcenter.local":
log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
is_vcenter_set = False
log.debug("vCenter data collection is enabled")
vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug")
# start the threads
auth_thread = start_thread(ZvmAuthHandler)
data_thread = start_thread(GetDataFunc)
#stats_thread = start_thread(GetStatsFunc())
vra_metrics_thread = start_thread(GetVraMetrics)
webserver_thread = start_thread(WebServer)
probe_thread = start_thread(ThreadProbe)
# loop indefinitely
while True:
# check if any thread has crashed
sleep(10)
if not probe_thread.is_alive():
# restart the thread
log.error("Probe Thread Died - Restarting")
probe_thread = start_thread(ThreadProbe)
if not auth_thread.is_alive():
# restart the thread
log.error("Authentication Thread Died - Restarting")
auth_thread = start_thread(ZvmAuthHandler)
if not data_thread.is_alive():
# restart the thread
log.error("Data Thread Died - Restarting")
data_thread = start_thread(GetDataFunc)
#if not stats_thread.is_alive():
# # restart the thread
# log.error("Stats Thread Died - Restarting")
# stats_thread = start_thread(GetStatsFunc())
if not vra_metrics_thread.is_alive():
# restart the thread
log.error("VRA Metrics Thread Died - Restarting")
vra_metrics_thread = start_thread(GetVraMetrics)
if not webserver_thread.is_alive():
# restart the thread
log.error("Webserver Thread Died - Restarting")
webserver_thread = start_thread(WebServer)
sleep(api_timeout)
+343 -449
View File
@@ -9,7 +9,7 @@ import threading
import socket
from pyVim.connect import SmartConnect, Disconnect
from pyVmomi import vim
from time import sleep
from time import sleep, time
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.structures import CaseInsensitiveDict
from tinydb import TinyDB, Query
@@ -18,8 +18,15 @@ from version import VERSION
from vmware.vcenter import vcsite
from zvma10.zvma import zvmsite
from posthog import Posthog
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
global start_time
start_time = time()
"""
Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually
"""
callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f')
verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't')
zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60')
@@ -28,245 +35,119 @@ client_id = os.environ.get('CLIENT_ID', 'api-script')
client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk')
scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30))
api_timeout = int(os.environ.get('API_TIMEOUT', 5))
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
LOGLEVEL = os.environ.get('LOGLEVEL', 'DEBUG').upper()
DISABLE_STATS = os.environ.get('DISABLE_STATS', 'FALSE').upper()
version = str(VERSION)
vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50')
vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local')
vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!')
# Get the hostname of the machine
container_id = str(socket.gethostname())
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
log = logging.getLogger("Node-Exporter")
log.setLevel(LOGLEVEL)
log.addHandler(log_handler)
log.info(f"Zerto-Node-Exporter - Version {version}")
log.info(f"Log Level: {LOGLEVEL}")
log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret)
# Global Variables
token = ""
siteId = "NotSet"
siteName = "NotSet"
siteZvmVersion = ""
siteVcVersion = ""
siteZvmMajorVersion = ""
siteZvmMinorVersion = ""
siteZvmUpdateVersion = ""
siteZvmPatchVersion = ""
lastStats = CaseInsensitiveDict()
# Check if vCenter is set, if not disable VRA metrics
is_vcenter_set = True
if vcenter_host == "vcenter.local":
log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
is_vcenter_set = False
log.debug("vCenter data collection is enabled")
vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug")
# Authentication Thread which handles authentication and token refresh for ZVM API
def ZvmAuthHandler():
log.debug("ZVMAuthHandler Thread Started")
expiresIn = 0
global token
global siteId
global siteName
retries = 0
while True:
if expiresIn < 30:
h = CaseInsensitiveDict()
h["Content-Type"] = "application/x-www-form-urlencoded"
d = CaseInsensitiveDict()
d["client_id"] = client_id
d["client_secret"] = client_secret
d["grant_type"] = "client_credentials"
uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token"
delay = 0
try:
response = requests.post(url=uri, data=d, headers=h, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
if 'access_token' not in responseJSON or 'expires_in' not in responseJSON:
log.error("Authentication response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
token = str(responseJSON.get('access_token'))
expiresIn = int(responseJSON.get('expires_in'))
log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds")
if response.status_code != 200:
log.error("Authentication request failed with status code " + str(response.status_code))
delay = 2 ** retries
sleep(delay)
retries += 1
continue
# Get Site ID and Name
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite"
delay = 0
try:
log.debug("Getting Site ID and Name")
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
response.raise_for_status()
except requests.exceptions.RequestException as e:
retries += 1
delay = 2 ** retries
log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds")
sleep(delay)
continue
else:
retries = 0
responseJSON = response.json()
log.debug(responseJSON)
if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON:
log.error("LocalSite API response does not contain expected keys")
delay = 2 ** retries
sleep(delay)
retries += 1
continue
else:
siteId = str(responseJSON.get('SiteIdentifier'))
siteName = str(responseJSON.get('SiteName'))
siteZvmVersion = str(responseJSON.get('Version'))
siteVcVersion = str(responseJSON.get('SiteTypeVersion'))
# Break out ZVM version strings
siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".")
siteZvmUpdateVersion = siteZvmUpdateVersion[0]
if (len(siteZvmUpdateVersion) > 1):
siteZvmPatchVersion = siteZvmUpdateVersion[1]
else:
siteZvmPatchVersion = "0"
log.info("Site ID: " + siteId + " Site Name: " + siteName)
expiresIn -= 10 + delay
log.debug("Token Expires in " + str(expiresIn) + " seconds")
sleep(10)
'''
# Thread which gets VM level encryption statistics from ZVM API
def GetStatsFunc():
def GetStatsFunc(zvm_instance):
tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging
dbvm = Query()
dbvpg = Query()
zvm = zvm_instance
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.info("Got Auth Token!")
log.debug("token: " + str(token))
if (zvm.is_authenticated()):
log.debug("Stats Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
## Statistics API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/"
statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
statsapi_json = statsapi.json()
#log.debug(statsapi_json)
statsapi_json = None
statsapi_json = zvm.vms_statistics()
log.debug(statsapi_json)
vms_encryption_metrics = zvm.encryptiondetection_metrics_vms()
for vm in statsapi_json:
oldvmdata = dict()
if statsapi_json is not None:
for vm in statsapi_json:
vmsiteinfo = zvm.vm(vmidentifier=vm['VmIdentifier'], vpgidentifier=vm['VpgIdentifier'])
if vmsiteinfo['ProtectedSite']['identifier'] == zvm.site_id:
log.debug(f"VM is protected at this site - {vm['VmIdentifier']}")
oldvmdata = dict()
if 'EncryptionMetrics' not in vm:
vm['EncryptionMetrics'] = {}
vm['VmName'] = None
CurrentIops = 0
CurrentWriteCounterInMBs = 0
CurrentSyncCounterInMBs = 0
CurrentNetworkTrafficCounterInMBs = 0
CurrentEncryptedLBs = 0
CurrentUnencryptedLBs = 0
CurrentTotalLBs = 0
CurrentPercentEncrypted = 0
VMName = "NA"
CurrentIops = 0
CurrentWriteCounterInMBs = 0
CurrentSyncCounterInMBs = 0
CurrentNetworkTrafficCounterInMBs = 0
CurrentEncryptedLBs = 0
CurrentUnencryptedLBs = 0
CurrentTotalLBs = 0
CurrentPercentEncrypted = 0
CurrentTrendChangeLevel = 0
VMName = "NA"
oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
for vmem in vms_encryption_metrics:
if vmem['Link']['identifier'] == vm['VmIdentifier']:
log.debug(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}")
#print(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}")
vm['EncryptionMetrics']['EncryptedData'] = vmem['EncryptionMetrics']['EncryptedData']
vm['EncryptionMetrics']['NonEncryptedData'] = vmem['EncryptionMetrics']['NonEncryptedData']
vm['EncryptionMetrics']['TrendChangeLevel'] = vmem['EncryptionMetrics']['TrendChangeLevel']
vm['VmName'] = vmem['Link']['name']
log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
if (oldvmdata):
log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
log.debug(oldvmdata[0])
log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
if (oldvmdata):
log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
log.debug(oldvmdata[0])
log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
log.debug("!@!@!@!@!@ Stats !@!@!@!@!@")
VMName = oldvmdata[0]['VmName']
log.debug("Current VM " + str(VMName))
CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter'])
log.debug("CurrentIops " + str(CurrentIops))
CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs'])
log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs'])
log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs'])
log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
if CurrentTotalLBs != 0:
CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100)
log.debug("!@!@!@!@!@ Stats !@!@!@!@!@")
VMName = oldvmdata[0]['VmName']
log.debug("Current VM " + str(VMName))
CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter'])
log.debug("CurrentIops " + str(CurrentIops))
CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs'])
log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
CurrentWriteCounterInMBs = abs(vm['WriteCounterInMBs'] - oldvmdata[0]['WriteCounterInMBs'])
log.debug("CurrentWriteCounterInMBs " + str(CurrentWriteCounterInMBs))
CurrentEncryptedLBs = abs(vm['EncryptionMetrics']['EncryptedData'] - oldvmdata[0]['EncryptionMetrics']['EncryptedData'])
log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
CurrentUnencryptedLBs = abs(vm['EncryptionMetrics']['NonEncryptedData'] - oldvmdata[0]['EncryptionMetrics']['NonEncryptedData'])
log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
CurrentTrendChangeLevel = abs(vm['EncryptionMetrics']['TrendChangeLevel'] - oldvmdata[0]['EncryptionMetrics']['TrendChangeLevel'])
log.debug("CurrentTrendChangeLevel " + str(CurrentTrendChangeLevel))
CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
if CurrentTotalLBs != 0:
CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100)
else:
CurrentPercentEncrypted = 0
log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
else:
log.info(f"{vm['VmIdentifier']} - {vm['VmName']} - No Record Found, Inserting into DB")
#insert original VM record to tempdb
log.debug(tempdb.insert(vm))
# Store Calculated Metrics
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
#metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
#metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
#metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
#metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
else:
CurrentPercentEncrypted = 0
log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}")
#print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}")
else:
log.debug("No VMS in Stats API")
else:
log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB")
#insert original VM record to tempdb
log.debug(tempdb.insert(vm))
# update database with VM name, for easier display in Grafana Legends
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier']
try:
vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vapi_json = vapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
else:
log.debug("vapi_json: " + str(vapi_json))
tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier'])
log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName'])
VMName = vapi_json['VmName']
# Store Calculated Metrics
metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('statsmetrics', 'w')
@@ -288,134 +169,112 @@ def GetStatsFunc():
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
'''
# Function which retrieves stats from various ZVM APIs and stores them in a metrics file
def GetDataFunc():
def GetDataFunc(zvm_instance):
tempdb = TinyDB(storage=MemoryStorage)
dbvm = Query()
zvm = zvm_instance
while (True) :
global token
global siteId
global siteName
if (token != ""):
log.debug("Got Auth Token!")
log.debug("token: " + str(token))
if (zvm.is_authenticated()):
log.info("Data Collector Loop Running")
metricsDictionary = {}
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
### VPGs API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
vpg_json = service.json()
#log.debug(vpg_json)
for vpg in vpg_json :
metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"]
metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"]
metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"]
metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"]
metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"]
metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"]
metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"]
metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"]
metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"]
metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"]
metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"]
metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"]
metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"]
metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"]
vpg_json = None
vpg_json = zvm.vpgs()
if(vpg_json is not None):
log.debug("Got VPG JSON")
for vpg in vpg_json :
metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"]
metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"]
metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"]
metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"]
metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"]
metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"]
metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"]
metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"]
metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"]
metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"]
metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"]
metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"]
metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"]
metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"]
else:
log.debug("No VPGs Found")
### Datastores APIs
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/"
service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
ds_json = service.json()
#log.debug(ds_json)
for ds in ds_json :
log.debug(f"Processing {ds['DatastoreName']}")
metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"]
metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"]
metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"]
metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"]
metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"]
metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"]
metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"]
ds_json = None
ds_json = zvm.datastores()
if(ds_json is not None):
log.debug("Got Datastores API")
for ds in ds_json :
log.debug(f"Processing {ds['DatastoreName']}")
metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"]
metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"]
metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"]
metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"]
metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"]
metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"]
metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"]
metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"]
else:
log.debug("No Datastores Found")
## VMs API
log.debug("Getting VMs API")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/"
scratch_vols = None
scratch_vols = zvm.vms()
if(scratch_vols is not None):
log.debug("Got VMs API")
for vm in scratch_vols:
log.debug("Processing VM: " + str(vm['VmName']))
log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId)
vmapi_json = {}
try:
vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
vmapi_json = vmapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
if siteId == vm['ProtectedSite']['identifier']:
log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site")
log.debug("Got VMs API")
log.debug(vmapi_json)
for vm in vmapi_json :
log.debug("Processing VM: " + str(vm['VmName']))
log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId)
if not isinstance(vm["ActualRPO"], int):
vm["ActualRPO"] = -1
metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"]
metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"]
metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"]
metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"]
metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"]
metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"]
metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"]
metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"]
metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"]
metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"]
metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"]
log.debug("Processed VM: " + str(vm['VmName']))
if siteId == vm['ProtectedSite']['identifier']:
log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site")
if not isinstance(vm["ActualRPO"], int):
vm["ActualRPO"] = -1
metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"]
metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"]
metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"]
metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"]
metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"]
metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"]
metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"]
metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"]
metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"]
metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"]
metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"]
log.debug("Processed VM: " + str(vm['VmName']))
else:
log.debug("VM " + vm['VmIdentifier'] + " is protected to this site")
else:
log.debug("VM " + vm['VmIdentifier'] + " is protected to this site")
else:
log.debug("No VMs Found")
## Volumes API for Scratch Volumes
log.debug("Getting Scratch Volumes")
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch"
scratch_vols = None
scratch_vols = zvm.volumes(volumetype="scratch")
volapi_json = {}
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got Scratch Volumes API")
if(bool(volapi_json)):
for volume in volapi_json :
if(scratch_vols is not None):
log.debug("Got Scratch Volumes API")
for volume in scratch_vols:
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"
@@ -426,24 +285,17 @@ def GetDataFunc():
percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100)
percentage_used = round(percentage_used, 1)
#metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used
else:
log.debug("No Scratch Volumes Found")
## Volumes API for Journal Volumes
log.debug("Getting Journal Volumes")
journal_vols = None
journal_vols = zvm.volumes(volumetype="journal")
volapi_json = {}
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal"
try:
volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
volapi_json = volapi.json()
except Exception as e:
log.error("Error while sending api request: " + str(e))
VMName = "Unknown"
return
log.debug("Got Journal Volumes API")
if(bool(volapi_json)):
if(journal_vols is not None):
log.debug("Journal Volumes Exist")
for volume in volapi_json :
for volume in journal_vols :
log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...")
#metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"]
# Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key.
@@ -464,6 +316,8 @@ def GetDataFunc():
metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1
else:
metricsDictionary[metrickey] = 1
else:
log.debug("No Journal Volumes Exist")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
log.debug("Writing metrics to file")
@@ -491,101 +345,89 @@ def GetDataFunc():
sleep(1)
# get VRA CPU and memory usage from vCenter Server
def GetVraMetrics():
# set up API endpoint and headers
log.debug("GetVraCpuMemory() called")
metricsDictionary = {}
while True:
vra_names = []
vras = []
global token
global siteId
global siteName
def GetVraMetrics(zvm_instance):
log.debug("GetVraMetrics thread started")
try:
log.debug("Checking Token in VRA CPU MEM Collector")
if (token != ""):
log.debug("Auth Token Valid!")
log.debug("token: " + str(token))
log.info("VRA CPU MEM Collector Running")
metricsDictionary = {}
zvm = zvm_instance
while True:
vra_names = []
vras = []
global siteId
global siteName
h2 = CaseInsensitiveDict()
h2["Accept"] = "application/json"
h2["Authorization"] = "Bearer " + token
log.debug("Checking Token in VRA CPU MEM Collector")
if (zvm.is_authenticated()):
log.info("VRA CPU MEM Collector Running")
### VRA API
uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras"
# make API call to get list of VRAs
try:
response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
except Exception as e:
log.error(f"Error connecting to {uri}: {e}")
return
else:
log.debug("Response from GET /v1/vras: %s", response.text)
# parse JSON response and get the name of each VRA
vras = response.json()
log.debug("VRA names: %s", vras)
log.debug(type(vras))
for vra in vras :
# Gather other VRA Metrics from Zerto API into Metrics Diectionary
metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"]
metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"]
metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"]
metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"]
metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"]
metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"]
metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"]
metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"]
metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"]
log.debug("VRA Name: %s", vra['VraName'])
log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}")
# get the CPU and memory usage for each VRA
if is_vcenter_set:
log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs")
try:
log.debug("Trying to get stats from vc module")
vradata = vc_connection.get_cpu_mem_used(vra['VraName'])
### VRA API
vras_json = None
vras_json = zvm.vras()
log.debug(vras_json)
# get the CPU usage and memory usage for the VM
cpu_usage_mhz = vradata[0]
memory_usage_mb = vradata[1]
if (vras_json is not None):
log.debug("VRA names: %s", vras_json)
log.debug(type(vras))
for vra in vras_json :
# Gather other VRA Metrics from Zerto API into Metrics Diectionary
metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"]
metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"]
metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"]
metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"]
metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"]
metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"]
metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"]
metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"]
metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"]
# print the CPU and memory usage for the VM
log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz
metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb
except:
log.info(f"No VM found with name {vra['VraName']}")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('vrametrics', 'w')
txt_object = open('vrametrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
log.debug("VRA Name: %s", vra['VraName'])
log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user}")
# This function will get data every 10 seconds
log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds")
sleep(scrape_speed * 2)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
# get the CPU and memory usage for each VRA
if is_vcenter_set:
log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs")
try:
log.debug("Trying to get stats from vc module")
vradata = vc_connection.get_cpu_mem_used(vra['VraName'])
# get the CPU usage and memory usage for the VM
cpu_usage_mhz = vradata[0]
memory_usage_mb = vradata[1]
# print the CPU and memory usage for the VM
log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz
metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb
except:
log.info(f"No VM found with name {vra['VraName']}")
else:
log.debug("No VRAs Found")
## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
file_object = open('vrametrics', 'w')
txt_object = open('vrametrics.txt', 'w')
for item in metricsDictionary :
file_object.write(item)
file_object.write(" ")
file_object.write(str(metricsDictionary[item]))
file_object.write("\n")
txt_object.write(item)
txt_object.write(" ")
txt_object.write(str(metricsDictionary[item]))
txt_object.write("\n")
file_object.close()
txt_object.close()
# This function will get data every 10 seconds
log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds")
sleep(scrape_speed * 2)
else:
log.debug("Waiting 1 second for Auth Token")
sleep(1)
except Exception as e:
log.error(f"Error in GetVraMetrics: {e}")
# function which monitors the threads and restarts them if they die
def ThreadProbe():
@@ -594,28 +436,27 @@ def ThreadProbe():
log.debug("Thread Probe Started")
metricsDictionary = {}
log.debug("Is Auth Thread Alive")
if auth_thread.is_alive():
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is Data Thread Alive")
uptime = round((time() - start_time) / 60, 1)
metricsDictionary["exporter_uptime{ExporterInstance=\"" + container_id + "\"}"] = uptime
if data_thread.is_alive():
log.debug("Data Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("Data Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
#log.debug("Is Stats Thread Alive")
#if stats_thread.is_alive():
# metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
#else:
# metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
if stats_thread.is_alive():
log.debug("Stats Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("Stats Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Is VRA Metrics Thread Alive")
if vra_metrics_thread.is_alive():
log.debug("VRA Metrics Thread Is Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
else:
log.debug("VRA Metrics Thread Is NOT Alive")
metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
log.debug("Writing Probe data to files")
@@ -638,7 +479,6 @@ def ThreadProbe():
log.debug("Probe Thread Going to Sleep")
sleep(30)
#----------------run http server on port 9999-----------------
def WebServer():
log.info("Web Server Started")
@@ -651,17 +491,75 @@ def WebServer():
httpd.serve_forever()
def start_thread(target_func):
# start a new thread
log.debug(f"Starting thread for {target_func.__name__}")
thread = threading.Thread(target=target_func)
thread.daemon = True
thread.start()
# return the thread object
log.debug(f"Thread {target_func.__name__} started")
return thread
# start the threads
auth_thread = start_thread(ZvmAuthHandler)
data_thread = start_thread(GetDataFunc)
#stats_thread = start_thread(GetStatsFunc())
vra_metrics_thread = start_thread(GetVraMetrics)
"""
Main Program Logic
"""
# Get the hostname of the machine
container_id = str(socket.gethostname())
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
log = logging.getLogger("Node-Exporter")
log.setLevel(LOGLEVEL)
log.addHandler(log_handler)
log.info(f"Zerto-Node-Exporter - Version {version}")
log.info(f"Log Level: {LOGLEVEL}")
log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret)
# Initialize zvmsite instance
zvm_instance = zvmsite(
host=zvm_url,
port=zvm_port,
client_id=client_id,
client_secret=client_secret,
grant_type="client_credentials",
loglevel=LOGLEVEL,
logger=log,
stats=DISABLE_STATS
)
# Start the zvmsite authentication thread
zvm_instance.connect()
"""
Global Variables used by the program
"""
local_site_info = None
siteId = None
siteName = None
while(siteId is None):
if zvm_instance.is_authenticated():
sleep(2)
log.debug("Trying Set Global Vars")
siteId = zvm_instance.site_id
siteName = zvm_instance.site_name
lastStats = CaseInsensitiveDict()
# Check if vCenter is set, if not disable VRA metrics
is_vcenter_set = True
if vcenter_host == "vcenter.local":
log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
is_vcenter_set = False
log.debug("vCenter data collection is enabled")
vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug", logger=log)
# Starting threads
vra_metrics_thread = start_thread(lambda: GetVraMetrics(zvm_instance))
data_thread = start_thread(lambda: GetDataFunc(zvm_instance))
stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance))
log.debug("Starting VRA Metrics")
webserver_thread = start_thread(WebServer)
probe_thread = start_thread(ThreadProbe)
@@ -673,22 +571,18 @@ while True:
# restart the thread
log.error("Probe Thread Died - Restarting")
probe_thread = start_thread(ThreadProbe)
if not auth_thread.is_alive():
# restart the thread
log.error("Authentication Thread Died - Restarting")
auth_thread = start_thread(ZvmAuthHandler)
if not data_thread.is_alive():
# restart the thread
log.error("Data Thread Died - Restarting")
data_thread = start_thread(GetDataFunc)
#if not stats_thread.is_alive():
# # restart the thread
# log.error("Stats Thread Died - Restarting")
# stats_thread = start_thread(GetStatsFunc())
data_thread = start_thread(GetDataFunc(zvm_instance))
if not stats_thread.is_alive():
# restart the thread
log.error("Stats Thread Died - Restarting")
stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance))
if not vra_metrics_thread.is_alive():
# restart the thread
log.error("VRA Metrics Thread Died - Restarting")
vra_metrics_thread = start_thread(GetVraMetrics)
vra_metrics_thread = start_thread(GetVraMetrics(zvm_instance))
if not webserver_thread.is_alive():
# restart the thread
log.error("Webserver Thread Died - Restarting")
+1 -1
View File
@@ -1,4 +1,4 @@
print("Initializing zvma10 package...")
print("Initializing vcenter package...")
#from .zvma import zvm
from .vcenter import vcsite
Binary file not shown.
Binary file not shown.
+15 -9
View File
@@ -4,10 +4,11 @@ from pyVmomi import vim, vmodl
import ssl
import datetime
import logging
import socket
from logging.handlers import RotatingFileHandler
class vcsite:
def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO"):
def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO", logger=None):
self.host = host
self.port = port
self.username = username
@@ -16,21 +17,26 @@ class vcsite:
self.version = None
self.__conn__ = None
self.LOGLEVEL = loglevel.upper()
self.log = None
#set log line format including container_id
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
self.log = logging.getLogger("Node-Exporter")
self.log.setLevel(self.LOGLEVEL)
self.log.addHandler(log_handler)
if logger is None:
#set log line format including container_id
container_id = str(socket.gethostname())
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
self.log = logging.getLogger("vCenter Module")
self.log.setLevel(self.LOGLEVEL)
self.log.addHandler(log_handler)
else:
self.log = logger
def connect(self):
self.log.info(f"Log Level set to {self.LOGLEVEL}")
if self.__conn__ is None:
context = ssl.create_default_context()
if not self.verify_ssl:
print("dont verify SSL")
log.debug("dont verify SSL")
# Create an SSL context without certificate verification
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
+517 -148
View File
@@ -12,6 +12,8 @@ from urllib3.exceptions import InsecureRequestWarning
from urllib.parse import urlencode
from urllib.parse import urlparse
from time import sleep
from datetime import datetime
from dateutil import parser
from typing import List, Dict, Tuple, Union, Any, Optional
from requests.structures import CaseInsensitiveDict
from logging.handlers import RotatingFileHandler
@@ -21,7 +23,7 @@ from requests import Request, Session
from .version import VERSION
class zvmsite:
def __init__(self, host, username=None, password=None, port: int = 443, verify_ssl: bool = False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug", stats: bool = True):
def __init__(self, host, username=None, password=None, port: int = 443, verify_ssl: bool = False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug", logger=None, stats: bool = True) -> None:
self.stats = stats
self.host = host
self.port = port
@@ -61,7 +63,11 @@ class zvmsite:
self._running = False
self.LOGLEVEL = loglevel.upper()
self.setup_logging()
if logger is None:
self.setup_logging()
else:
self.log = logger
atexit.register(self.disconnect)
self._running = True
@@ -74,7 +80,7 @@ class zvmsite:
self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded')
self.log.debug("Sent PostHog Hook")
def __authhandler__(self):
def __authhandler__(self) -> None:
self.log.info(f"Log Level set to {self.LOGLEVEL}")
if not self.__connected__:
context = ssl.create_default_context()
@@ -110,6 +116,10 @@ class zvmsite:
self.expiresIn = int(response['expires_in'])
self.log.info("Authentication successful")
self.__connected__ = True
local_site_info = self.local_site()
self.site_id = local_site_info['SiteIdentifier']
self.site_name = local_site_info['SiteName']
else:
self.log.error("Authentication failed")
sleep(2 ** retries)
@@ -131,7 +141,7 @@ class zvmsite:
log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5)
log_handler.setFormatter(log_formatter)
self.log = logging.getLogger("Node-Exporter")
self.log = logging.getLogger("ZVM10 Module")
self.log.setLevel(self.LOGLEVEL)
self.log.addHandler(log_handler)
@@ -322,6 +332,23 @@ class zvmsite:
return False # Return False if the try block didn't execute successfully
def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None,
entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]:
params = {
'startdate': startdate,
'enddate': enddate,
'vpgid': vpgid,
'zorgidentifier': zorgidentifier,
'level': level,
'entity': entity,
'helpidentifier': helpidentifier,
'isdismissed': isdismissed
}
uri = self.construct_url("v1/alerts", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def alert_levels(self) -> List[str]:
params = {
@@ -345,25 +372,8 @@ class zvmsite:
uri = self.construct_url(f"v1/alerts/helpidentifiers", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None,
entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]:
params = {
'startdate': startdate,
'enddate': enddate,
'vpgid': vpgid,
'zorgidentifier': zorgidentifier,
'level': level,
'entity': entity,
'helpidentifier': helpidentifier,
'isdismissed': isdismissed
}
uri = self.construct_url("v1/alerts", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def datastore(self, datastoreidentifier=None):
def datastore(self, datastoreidentifier=None) -> Dict[str, Any]:
if datastoreidentifier is None:
self.log.error("Datastore identifier is required for get_datastore function.")
@@ -382,6 +392,59 @@ class zvmsite:
uri = self.construct_url("v1/datastores", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def datetime_local(self) -> datetime:
params = {}
uri = self.construct_url(f"v1/serverDateTime/serverDateTimeLocal", {})
response = self.make_api_request("GET", uri, headers=self.apiheader)
if response is not None:
# Extract the datetime string from the JSON response
return parser.isoparse(response)
else:
error_message = "API request failed or returned None"
self.log.error(error_message)
raise ValueError(error_message)
def datetime_utc(self,) -> datetime:
params = {}
uri = self.construct_url(f"v1/serverDateTime/serverDateTimeUtc", params)
response = self.make_api_request("GET", uri, headers=self.apiheader)
if response is not None:
# Extract the datetime string from the JSON response
return parser.isoparse(response)
else:
error_message = "API request failed or returned None"
self.log.error(error_message)
raise ValueError(error_message)
def datetime_check(self, dt_str: str) -> datetime:
try:
# Try to parse the string into a datetime object
dt = parser.isoparse(dt_str)
except ValueError:
# If parsing fails, raise an error
raise ValueError("The 'dt_str' parameter must be a valid datetime string.")
# Format the datetime object for the API call
formatted_datetime = dt.isoformat()
params = {'datetime': formatted_datetime}
# Construct the URL with the datetime argument
uri = self.construct_url(f"v1/serverDateTime/dateTimeArgument", params)
# Make the API request
response = self.make_api_request("GET", uri, headers=self.apiheader)
# Check if the response is not None and parse the datetime string
if response is not None:
return parser.isoparse(response)
else:
error_message = "API request failed or returned None"
self.log.error(error_message)
raise requests.exceptions.HTTPError(error_message)
def encryptiondetection_enable(self):
@@ -450,7 +513,7 @@ class zvmsite:
uri = self.construct_url("v1/encryptionDetection/suspected/vpgs", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event(self, eventidentifier=None):
def event(self, eventidentifier=None) -> Dict[str, Any]:
if eventidentifier is None:
self.log.error("Event identifier is required for get event function.")
@@ -461,30 +524,6 @@ class zvmsite:
uri = self.construct_url(f"v1/events/{eventidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_types(self):
params = {
}
uri = self.construct_url(f"v1/events/types", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_entities(self):
params = {
}
uri = self.construct_url(f"v1/events/entities", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_categories(self):
params = {
}
uri = self.construct_url(f"v1/events/categories", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def events(self, startdate=None, enddate=None, vpgid=None, sitename=None, zorgidentifier=None, eventtype=None,
entitytype=None, category=None, username=None, alertidentifier=None) -> List[Dict[str, Any]]:
@@ -504,8 +543,74 @@ class zvmsite:
uri = self.construct_url("v1/events", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_types(self) -> List[str]:
params = {
}
uri = self.construct_url(f"v1/events/types", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_entities(self) -> List[str]:
params = {
}
uri = self.construct_url(f"v1/events/entities", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def event_categories(self) -> List[str]:
params = {
}
uri = self.construct_url(f"v1/events/categories", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def license(self) -> Dict[str, Any]:
params = {
}
uri = self.construct_url(f"v1/license", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def local_site(self):
def license_delete(self) -> bool:
params = {}
uri = self.construct_url(f"v1/license", params)
try:
response = self.make_api_request("DELETE", uri, headers=self.apiheader)
# Check if the response status code is 200 (OK)
if response.status_code == 200:
return True
else:
# Log and raise an exception for any non-200 status codes
self.log.error(f"Failed to delete license: {response.status_code}")
response.raise_for_status()
except requests.exceptions.RequestException as e:
self.log.error(f"Error while sending license delete request: {e}")
raise
return False # Return False if the try block didn't execute successfully
def license_apply(self, license=None):
if license is None:
self.log.error("A license key is required for apply license function.")
raise ValueError("License key is required.")
params = {
}
license = {
"licenseKey": license
}
uri = self.construct_url(f"v1/license", params)
return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader)
def local_site(self) -> Dict[str, Any]:
params = {
}
@@ -521,15 +626,26 @@ class zvmsite:
uri = self.construct_url(f"v1/localsite/pairingstatuses", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def local_site_send_billing(self):
params = {
}
def local_site_send_billing(self) -> bool:
params = {}
uri = self.construct_url(f"v1/localsite/settings/sendusage", params)
return self.make_api_request("POST", uri, headers=self.apiheader)
def local_site_banner(self):
try:
response = self.make_api_request("POST", uri, headers=self.apiheader)
# Check if the response status code is 200 (OK)
if response.status_code == 200:
return True
else:
# Log and raise an exception for any non-200 status codes
self.log.error(f"Failed to send billing information: {response.status_code}")
response.raise_for_status()
except requests.exceptions.RequestException as e:
self.log.error(f"Error while sending billing information request: {e}")
raise
return False # Return False if the try block didn't execute successfully
def local_site_banner(self) -> Dict[str, Any]:
params = {
}
@@ -549,40 +665,7 @@ class zvmsite:
# uri is spelled incorrectly because it is also spelled incorrectly in zerto
uri = self.construct_url(f"v1/localsite/settings/logingbanner", params)
return self.make_api_request("PUT", uri, json_data=data, headers=self.apiheader)
def license(self):
params = {
}
uri = self.construct_url(f"v1/license", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def license_delete(self):
params = {
}
uri = self.construct_url(f"v1/license", params)
return self.make_api_request("DELETE", uri, headers=self.apiheader)
def license_apply(self, license=None):
if license is None:
self.log.error("A license key is required for apply license function.")
raise ValueError("License key is required.")
params = {
}
license = {
"licenseKey": license
}
uri = self.construct_url(f"v1/license", params)
return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader)
def peer_sites(self) -> List[Dict[str, Any]]:
params = {
@@ -591,7 +674,7 @@ class zvmsite:
uri = self.construct_url(f"v1/peersites", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def peer_site(self, siteidentifier=None):
def peer_site(self, siteidentifier=None) -> Dict[str, Any]:
if siteidentifier is None:
self.log.error("Site identifier is required for get site function.")
raise ValueError("Site identifier is required.")
@@ -602,7 +685,7 @@ class zvmsite:
uri = self.construct_url(f"v1/peersites/{siteidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def peer_sites_pairing_statues(self):
def peer_sites_pairing_statues(self) -> List[str]:
params = {
}
@@ -644,12 +727,83 @@ class zvmsite:
uri = self.construct_url(f"v1/peersites/{siteidentifier}", params)
return self.make_api_request("DELETE", uri, json=data, headers=self.apiheader)
def peer_site_pairing_token(self):
def peer_site_pairing_token(self) -> Dict[str, Any]:
params = {}
uri = self.construct_url(f"v1/peersites/generatetoken", params)
return self.make_api_request("POST", uri, headers=self.apiheader)
def recovery_reports(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, vpgname=None, recoverytype=None, state=None) -> List[Dict[str, Any]]:
params = {
'starttime': starttime,
'endtime': endtime,
'pagenumber': pagenumber,
'pagesize': pagesize,
'vpgname': vpgname,
'recoverytype': recoverytype,
'state': state
}
uri = self.construct_url("v1/reports/recovery", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def recovery_report(self, recoveryoperationidentifier=None) -> Dict[str, Any]:
if recoveryoperationidentifier is None:
self.log.error("RecoveryOperationIdentifier is required for function.")
raise ValueError("RecoveryOperationIdentifier is required.")
params = {}
uri = self.construct_url(f"v1/reports/recovery/{recoveryoperationidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def resources_report(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, zorgname=None, vpgname=None, vmname=None,
protectedsitename=None, protectedclustername=None, protectedhostname=None, protectedorgvdc=None, protectedvcdorg=None, recoverysitename=None,
recoveryclustername=None, recoveryhostname=None, recoveryorgvdc=None, recoveryvcdorg=None) -> List[Dict[str, Any]]:
params = {
'starttime': starttime,
'endtime': endtime,
'pagenumber': pagenumber,
'pagesize': pagesize,
'vpgname': vpgname,
'vmname': vmname,
'protectedsitename': protectedsitename,
'protectedclustername': protectedclustername,
'protectedhostname': protectedhostname,
'protectedorgvdc': protectedorgvdc,
'protectedvcdorg': protectedvcdorg,
'recoverysitename': recoverysitename,
'recoveryclustername': recoveryclustername,
'recoveryhostname': recoveryhostname,
'recoveryorgvdc': recoveryorgvdc,
'recoveryvcdorg': recoveryvcdorg
}
uri = self.construct_url("v1/reports/resources", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def service_profiles(self) -> List[Dict[str, Any]]:
params = {
}
uri = self.construct_url(f"/v1/serviceprofiles", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def service_profile(self, serviceProfileIdentifier=None) -> Dict[str, Any]:
if siteidentifier is None:
self.log.error("Service Profile identifier is required for get site function.")
raise ValueError("Service Profile identifier is required.")
params = {
}
uri = self.construct_url(f"/v1/serviceprofiles/{serviceProfileIdentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None) -> List[Dict[str, Any]]:
params = {
@@ -664,7 +818,7 @@ class zvmsite:
uri = self.construct_url("v1/tasks", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def task(self, taskidentifier=None):
def task(self, taskidentifier=None) -> Dict[str, Any]:
if taskidentifier is None:
self.log.error("Task identifier is required for function.")
@@ -675,7 +829,7 @@ class zvmsite:
uri = self.construct_url(f"v1/tasks/{taskidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def task_types(self):
def task_types(self) -> List[str]:
params = {
}
@@ -683,56 +837,13 @@ class zvmsite:
uri = self.construct_url(f"v1/tasks/types", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg(self, vpgidentifier=None):
def vms_statistics(self) -> List[Dict[str, Any]]:
if vpgidentifier is None:
self.log.error("Vpg identifier is required for get_vpg function.")
raise ValueError("VM identifier is required.")
params = {
}
params = { }
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params)
uri = self.construct_url("v1/statistics/vms", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None,
recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None,
zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]:
params = {
'vpgid': vpgid,
'vpgname': vpgname,
'vpgstatus': vpgstatus,
'vpgsubstatus': vpgsubstatus,
'protectedsitetype': protectedsitetype,
'recoverysitetype': recoverysitetype,
'protectedsiteidentifier': protectedsiteidentifier,
'recoverysiteidentifier': recoverysiteidentifier,
'zorgidentifier': zorgidentifier,
'priority': priority,
'serviceprofileidentifier': serviceprofileidentifier
}
uri = self.construct_url("v1/vpgs", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True):
if vpgidentifier is None:
self.log.error("VPG identifier is required for delete_vpg function.")
raise ValueError("VPG identifier is required.")
# URL with vpgidentifier in the path
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}")
# Data to be sent in the request body
data = {
"keepRecoveryVolumes": keeprecoveryvolumes,
"force": force
}
# Make the POST request
return self.make_api_request("POST", uri, data=data, headers=self.apiheader)
def vms(self, vmidentifier=None, vmname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None,
recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None,
zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None) -> List[Dict[str, Any]]:
@@ -773,8 +884,8 @@ class zvmsite:
def vm_pointintime(self, vmidentifier=None, vpgidentifier=None, includebackupvms: bool = None, includemountedvms: bool = None):
if vmidentifier is None:
self.log.error("VM identifier is required for get_vm function.")
raise ValueError("VM identifier is required.")
self.log.error("VM identifier is required for vm_pointintime function.")
raise ValueError("VM identifier is required for vm_pointintime.")
params = {
'vpgidentifier': vpgidentifier,
@@ -783,8 +894,36 @@ class zvmsite:
}
uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsintime", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
stats = self.make_api_request("GET", uri, headers=self.apiheader)
if isinstance(stats, list) and not stats:
self.log.error("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.")
raise ValueError("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.")
elif stats is None:
self.log.error("VM not found, or vpgidentifier must be specified")
raise ValueError("VM not found, or vpgidentifier must be specified")
else:
return stats
def vm_pointintime_stats(self, vmidentifier=None, vpgidentifier=None):
if vmidentifier is None:
self.log.error("VM identifier is required for get_vm function.")
raise ValueError("VM identifier is required.")
params = {
'vpgidentifier': vpgidentifier
}
uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsInTime/stats", params)
stats = self.make_api_request("GET", uri, headers=self.apiheader)
if stats is None:
self.log.error("VM not found, or vpgidentifier must be specified")
raise ValueError("VM not found, or vpgidentifier must be specified")
else:
return stats
def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None) -> List[Dict[str, Any]]:
if volumetype:
valid_volumetypes = ["scratch", "journal", "recovery", "protected", "appliance"]
@@ -805,8 +944,238 @@ class zvmsite:
uri = self.construct_url("v1/volumes", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg(self, vpgidentifier=None) -> Dict[str, Any]:
if vpgidentifier is None:
self.log.error("Vpg identifier is required for get_vpg function.")
raise ValueError("VM identifier is required.")
params = {
}
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_checkpoints(self, vpgidentifier=None) -> Dict[str, Any]:
if vpgidentifier is None:
self.log.error("Vpg identifier is required for vpg_checkpoints function.")
raise ValueError("vpgidentifier is required.")
params = {
}
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_take_checkpoint(self, vpgidentifier=None, checkpointname=None) -> Dict[str, Any]:
if vpgidentifier is None:
self.log.error("Vpg identifier is required for vpg_checkpoints function.")
raise ValueError("vpgidentifier is required.")
# Construct the JSON payload
json_payload = {"checkpointname": "Checkpoint By Python ZVM Module"}
if checkpointname is not None:
json_payload["checkpointname"] = checkpointname
params = { }
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params)
return self.make_api_request("POST", uri, json_data=json_payload, headers=self.apiheader)
def vpg_checkpoint_stats(self, vpgidentifier=None) -> Dict[str, Any]:
if vpgidentifier is None:
self.log.error("Vpg identifier is required for vpg_checkpoints function.")
raise ValueError("vpgidentifier is required.")
params = {
}
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints/stats", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None,
recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None,
zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]:
params = {
'vpgid': vpgid,
'vpgname': vpgname,
'vpgstatus': vpgstatus,
'vpgsubstatus': vpgsubstatus,
'protectedsitetype': protectedsitetype,
'recoverysitetype': recoverysitetype,
'protectedsiteidentifier': protectedsiteidentifier,
'recoverysiteidentifier': recoverysiteidentifier,
'zorgidentifier': zorgidentifier,
'priority': priority,
'serviceprofileidentifier': serviceprofileidentifier
}
uri = self.construct_url("v1/vpgs", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def __set_zvm_version__(self):
def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True):
if vpgidentifier is None:
self.log.error("VPG identifier is required for delete_vpg function.")
raise ValueError("VPG identifier is required.")
# URL with vpgidentifier in the path
uri = self.construct_url(f"v1/vpgs/{vpgidentifier}")
# Data to be sent in the request body
data = {
"keepRecoveryVolumes": keeprecoveryvolumes,
"force": force
}
# Make the POST request
return self.make_api_request("POST", uri, data=data, headers=self.apiheader)
def vpg_retention_policies(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/retentionpolicies", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_priorities(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/priorities", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_entity_types(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/entitytypes", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_statuses(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/statuses", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_substatuses(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/substatuses", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_failover_shutdown_policies(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/failovershutdownpolicies", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vpg_failover_commit_policies(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vpgs/failovercommitpolicies", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vras(self, vraname=None, status=None, vraversion=None, hostname=None, ipaddress=None,
vragroup=None, datastorename=None, datastoreclustername=None, networkname=None, vraipconfigurationapi=None) -> List[Dict[str, Any]]:
params = {
'vraname': vraname,
'status': status,
'vraversion': vraversion,
'hostname': hostname,
'ipaddress': ipaddress,
'vragroup': vragroup,
'datastorename': datastorename,
'datastoreclustername': datastoreclustername,
'networkname': networkname,
'vraipconfigurationapi': vraipconfigurationapi
}
uri = self.construct_url(f"v1/vras", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vra(self, vraidentifier=None) -> Dict[str, Any]:
if vraidentifier is None:
self.log.error("vraidentifier is required for vra function.")
raise ValueError("vraidentifier is required.")
params = {
}
uri = self.construct_url(f"v1/vras/{vraidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vra_upgrade(self, vraidentifier=None) -> Dict[str, Any]:
if vraidentifier is None:
self.log.error("vraidentifier is required for vra function.")
raise ValueError("vraidentifier is required.")
params = {
}
uri = self.construct_url(f"v1/vras/{vraidentifier}/upgrade", params)
return self.make_api_request("POST", uri, headers=self.apiheader)
def vra_statuses(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vras/statuses", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def vra_ipconfigurationtypes(self) -> List[str]:
params = {}
uri = self.construct_url(f"v1/vras/ipconfigurationtypes", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
"""
def vra_cluster_settings(self, clusteridentifier=None) -> Dict[str, Any]:
if clusteridentifier is None:
self.log.error("clusteridentifier is required for vra function.")
raise ValueError("clusteridentifier is required.")
params = {
}
uri = self.construct_url(f"v1/vras/clusters/{clusteridentifier}/settings", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
"""
def zorgs(self) -> Dict[str, Any]:
params = {
}
uri = self.construct_url(f"v1/zorgs", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def zorg(self, zorgidentifier=None) -> Dict[str, Any]:
if zorgidentifier is None:
self.log.error("zorgidentifier is required for function.")
raise ValueError("zorgidentifier is required.")
params = {
}
uri = self.construct_url(f"v1/zorgs/{zorgidentifier}", params)
return self.make_api_request("GET", uri, headers=self.apiheader)
def __set_zvm_version__(self) -> None:
uri = self.construct_url("v1/localsite")
response = self.make_api_request("GET", uri, headers=self.apiheader)
if response:
@@ -824,7 +1193,7 @@ class zvmsite:
self.zvm_version['patch'] = temp[1] if len(temp) > 1 else "0"
self.log.info(f"Site ID: {self.site_id}, Site Name: {self.site_name}, Site Type: {self.site_type}")
def version(self):
def version(self) -> Dict[str, Any]:
if self.__connected__ and self._running:
if self.zvm_version['full'] is None:
self.__set_zvm_version__()