diff --git a/.gitignore b/.gitignore index 52fc19b..5b51a0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ logs/* .env/* .DS_Store +uuid.txt app/metrics app/metrics.txt app/statsmetrics diff --git a/app/exporter.py b/app/exporter.py deleted file mode 100644 index 4bfc477..0000000 --- a/app/exporter.py +++ /dev/null @@ -1,719 +0,0 @@ -import requests -import http.server -import socketserver -import os -import ssl -import logging -from logging.handlers import RotatingFileHandler -import threading -import socket -from pyVim.connect import SmartConnect, Disconnect -from pyVmomi import vim -from time import sleep -from requests.packages.urllib3.exceptions import InsecureRequestWarning -from requests.structures import CaseInsensitiveDict -from tinydb import TinyDB, Query -from tinydbstorage.storage import MemoryStorage -from version import VERSION -from vmware.vcenter import vcsite -from zvma10.zvma import zvmsite -from posthog import Posthog -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -""" -Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually -""" - -callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') -verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') -zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') -zvm_port = os.environ.get('ZVM_PORT', '443') -client_id = os.environ.get('CLIENT_ID', 'api-script') -client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') -scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) -api_timeout = int(os.environ.get('API_TIMEOUT', 5)) -LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() -version = str(VERSION) -vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') -vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') -vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') - - -""" -Global Variables used by the program -""" -token = "" -siteId = "NotSet" -siteName = "NotSet" -siteZvmVersion = "" -siteVcVersion = "" -siteZvmMajorVersion = "" -siteZvmMinorVersion = "" -siteZvmUpdateVersion = "" -siteZvmPatchVersion = "" -lastStats = CaseInsensitiveDict() - - -# Authentication Thread which handles authentication and token refresh for ZVM API -def ZvmAuthHandler(): - log.debug("ZVMAuthHandler Thread Started") - expiresIn = 0 - global token - global siteId - global siteName - retries = 0 - while True: - if expiresIn < 30: - h = CaseInsensitiveDict() - h["Content-Type"] = "application/x-www-form-urlencoded" - - d = CaseInsensitiveDict() - d["client_id"] = client_id - d["client_secret"] = client_secret - d["grant_type"] = "client_credentials" - - uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token" - delay = 0 - - try: - response = requests.post(url=uri, data=d, headers=h, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: - log.error("Authentication response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - token = str(responseJSON.get('access_token')) - expiresIn = int(responseJSON.get('expires_in')) - log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds") - - if response.status_code != 200: - log.error("Authentication request failed with status code " + str(response.status_code)) - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - # Get Site ID and Name - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite" - delay = 0 - try: - log.debug("Getting Site ID and Name") - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - log.debug(responseJSON) - if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: - log.error("LocalSite API response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - else: - siteId = str(responseJSON.get('SiteIdentifier')) - siteName = str(responseJSON.get('SiteName')) - siteZvmVersion = str(responseJSON.get('Version')) - siteVcVersion = str(responseJSON.get('SiteTypeVersion')) - - # Break out ZVM version strings - siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") - siteZvmUpdateVersion = siteZvmUpdateVersion[0] - if (len(siteZvmUpdateVersion) > 1): - siteZvmPatchVersion = siteZvmUpdateVersion[1] - else: - siteZvmPatchVersion = "0" - log.info("Site ID: " + siteId + " Site Name: " + siteName) - - expiresIn -= 10 + delay - log.debug("Token Expires in " + str(expiresIn) + " seconds") - sleep(10) - -''' -# Thread which gets VM level encryption statistics from ZVM API - -def GetStatsFunc(): - tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging - dbvm = Query() - dbvpg = Query() - while (True) : - global token - global siteId - global siteName - - if (token != ""): - log.info("Got Auth Token!") - log.debug("token: " + str(token)) - log.debug("Stats Collector Loop Running") - - metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - ## Statistics API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" - statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - statsapi_json = statsapi.json() - #log.debug(statsapi_json) - - for vm in statsapi_json: - oldvmdata = dict() - - CurrentIops = 0 - CurrentWriteCounterInMBs = 0 - CurrentSyncCounterInMBs = 0 - CurrentNetworkTrafficCounterInMBs = 0 - CurrentEncryptedLBs = 0 - CurrentUnencryptedLBs = 0 - CurrentTotalLBs = 0 - CurrentPercentEncrypted = 0 - VMName = "NA" - - oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) - - log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) - if (oldvmdata): - log.info(vm['VmIdentifier'] + " Record Found, Updating DB") - log.debug(oldvmdata[0]) - log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) - - log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") - VMName = oldvmdata[0]['VmName'] - log.debug("Current VM " + str(VMName)) - CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) - log.debug("CurrentIops " + str(CurrentIops)) - CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) - log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) - CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) - log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) - CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) - log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) - CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) - log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) - CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) - log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) - if CurrentTotalLBs != 0: - CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) - else: - CurrentPercentEncrypted = 0 - log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) - - else: - log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") - #insert original VM record to tempdb - log.debug(tempdb.insert(vm)) - - # update database with VM name, for easier display in Grafana Legends - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] - try: - vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vapi_json = vapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - else: - log.debug("vapi_json: " + str(vapi_json)) - tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) - log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) - VMName = vapi_json['VmName'] - - # Store Calculated Metrics - metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops - metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs - metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs - metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs - metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs - metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs - metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs - metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('statsmetrics', 'w') - txt_object = open('statsmetrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - file_object.close() - txt_object.close() - - log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") - sleep(scrape_speed) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) -''' - -# Function which retrieves stats from various ZVM APIs and stores them in a metrics file -def GetDataFunc(): - tempdb = TinyDB(storage=MemoryStorage) - dbvm = Query() - while (True) : - global token - global siteId - global siteName - - if (token != ""): - log.debug("Got Auth Token!") - log.debug("token: " + str(token)) - log.info("Data Collector Loop Running") - - metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - - ### VPGs API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - vpg_json = service.json() - #log.debug(vpg_json) - for vpg in vpg_json : - metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] - metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] - metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] - metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] - metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] - metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] - metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] - metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] - metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] - metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] - metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] - metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] - metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] - metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] - - ### Datastores APIs - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - ds_json = service.json() - #log.debug(ds_json) - for ds in ds_json : - - log.debug(f"Processing {ds['DatastoreName']}") - - metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] - metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] - metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] - metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] - metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] - metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] - metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] - - ## VMs API - log.debug("Getting VMs API") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" - - vmapi_json = {} - try: - vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vmapi_json = vmapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got VMs API") - log.debug(vmapi_json) - for vm in vmapi_json : - log.debug("Processing VM: " + str(vm['VmName'])) - log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) - - if siteId == vm['ProtectedSite']['identifier']: - log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - - if not isinstance(vm["ActualRPO"], int): - vm["ActualRPO"] = -1 - metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] - metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] - metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] - metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] - metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] - metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] - metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] - metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] - metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] - metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] - metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] - log.debug("Processed VM: " + str(vm['VmName'])) - - else: - log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") - - - ## Volumes API for Scratch Volumes - log.debug("Getting Scratch Volumes") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch" - - volapi_json = {} - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Scratch Volumes API") - if(bool(volapi_json)): - for volume in volapi_json : - #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] - # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. - metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] - percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100) - percentage_used = round(percentage_used, 1) - #metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used - - ## Volumes API for Journal Volumes - log.debug("Getting Journal Volumes") - - volapi_json = {} - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal" - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Journal Volumes API") - if(bool(volapi_json)): - log.debug("Journal Volumes Exist") - for volume in volapi_json : - log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...") - #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] - # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. - metrickey = "vm_journal_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] - - metrickey = "vm_journal_volume_provisioned_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["ProvisionedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["ProvisionedInBytes"] - - metrickey = "vm_journal_volume_count{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1 - else: - metricsDictionary[metrickey] = 1 - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - log.debug("Writing metrics to file") - file_object = open('metrics', 'w') - txt_object = open('metrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() - log.debug("Metrics written to file") - - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") - sleep(scrape_speed) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) - -# get VRA CPU and memory usage from vCenter Server -def GetVraMetrics(): - # set up API endpoint and headers - log.debug("GetVraCpuMemory() called") - metricsDictionary = {} - while True: - vra_names = [] - vras = [] - global token - global siteId - global siteName - - log.debug("Checking Token in VRA CPU MEM Collector") - if (token != ""): - log.debug("Auth Token Valid!") - log.debug("token: " + str(token)) - log.info("VRA CPU MEM Collector Running") - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - - ### VRA API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras" - - # make API call to get list of VRAs - try: - response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - except Exception as e: - log.error(f"Error connecting to {uri}: {e}") - return - else: - log.debug("Response from GET /v1/vras: %s", response.text) - # parse JSON response and get the name of each VRA - - vras = response.json() - - log.debug("VRA names: %s", vras) - log.debug(type(vras)) - for vra in vras : - # Gather other VRA Metrics from Zerto API into Metrics Diectionary - metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] - metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] - metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] - metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] - metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] - metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] - metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] - metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] - metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - - - log.debug("VRA Name: %s", vra['VraName']) - log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") - - # get the CPU and memory usage for each VRA - if is_vcenter_set: - log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") - try: - log.debug("Trying to get stats from vc module") - vradata = vc_connection.get_cpu_mem_used(vra['VraName']) - - # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vradata[0] - memory_usage_mb = vradata[1] - - # print the CPU and memory usage for the VM - log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") - metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz - metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb - except: - log.info(f"No VM found with name {vra['VraName']}") - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('vrametrics', 'w') - txt_object = open('vrametrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() - - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") - sleep(scrape_speed * 2) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) - -# function which monitors the threads and restarts them if they die -def ThreadProbe(): - global container_id - while True: - log.debug("Thread Probe Started") - metricsDictionary = {} - - log.debug("Is Auth Thread Alive") - if auth_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is Data Thread Alive") - if data_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - #log.debug("Is Stats Thread Alive") - #if stats_thread.is_alive(): - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - #else: - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is VRA Metrics Thread Alive") - if vra_metrics_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Writing Probe data to files") - file_object = open('threads', 'w') - txt_object = open('threads.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - log.debug("Trying to Close probe txt files") - file_object.close() - txt_object.close() - - log.debug("Probe Thread Going to Sleep") - sleep(30) - -#----------------run http server on port 9999----------------- -def WebServer(): - log.info("Web Server Started") - PORT = 9999 - - Handler = http.server.SimpleHTTPRequestHandler - - with socketserver.TCPServer(("", PORT), Handler) as httpd: - log.info(f"Webserver running on port {PORT}") - httpd.serve_forever() - -def start_thread(target_func): - # start a new thread - thread = threading.Thread(target=target_func) - thread.start() - # return the thread object - return thread - -""" -Main Program Logic -""" - -# Initialize zvmsite instance -zvm_instance = zvmsite( - host=zvm_url, - port=zvm_port, - client_id=client_id, - client_secret=client_secret, - grant_type="client_credentials", - loglevel=LOGLEVEL -) - -# Start the zvmsite authentication thread -zvm_instance.connect() - -# Get the hostname of the machine -container_id = str(socket.gethostname()) - -#set log line format including container_id -log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") -log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) -log_handler.setFormatter(log_formatter) -log = logging.getLogger("Node-Exporter") -log.setLevel(LOGLEVEL) -log.addHandler(log_handler) -log.info(f"Zerto-Node-Exporter - Version {version}") -log.info(f"Log Level: {LOGLEVEL}") -log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) - -# Check if vCenter is set, if not disable VRA metrics -is_vcenter_set = True -if vcenter_host == "vcenter.local": - log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") - is_vcenter_set = False -log.debug("vCenter data collection is enabled") -vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") - - -# start the threads -auth_thread = start_thread(ZvmAuthHandler) -data_thread = start_thread(GetDataFunc) -#stats_thread = start_thread(GetStatsFunc()) -vra_metrics_thread = start_thread(GetVraMetrics) -webserver_thread = start_thread(WebServer) -probe_thread = start_thread(ThreadProbe) - -# loop indefinitely -while True: - # check if any thread has crashed - sleep(10) - if not probe_thread.is_alive(): - # restart the thread - log.error("Probe Thread Died - Restarting") - probe_thread = start_thread(ThreadProbe) - if not auth_thread.is_alive(): - # restart the thread - log.error("Authentication Thread Died - Restarting") - auth_thread = start_thread(ZvmAuthHandler) - if not data_thread.is_alive(): - # restart the thread - log.error("Data Thread Died - Restarting") - data_thread = start_thread(GetDataFunc) - #if not stats_thread.is_alive(): - # # restart the thread - # log.error("Stats Thread Died - Restarting") - # stats_thread = start_thread(GetStatsFunc()) - if not vra_metrics_thread.is_alive(): - # restart the thread - log.error("VRA Metrics Thread Died - Restarting") - vra_metrics_thread = start_thread(GetVraMetrics) - if not webserver_thread.is_alive(): - # restart the thread - log.error("Webserver Thread Died - Restarting") - webserver_thread = start_thread(WebServer) - sleep(api_timeout) \ No newline at end of file diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index a6bc628..2cb941e 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -9,7 +9,7 @@ import threading import socket from pyVim.connect import SmartConnect, Disconnect from pyVmomi import vim -from time import sleep +from time import sleep, time from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.structures import CaseInsensitiveDict from tinydb import TinyDB, Query @@ -18,8 +18,15 @@ from version import VERSION from vmware.vcenter import vcsite from zvma10.zvma import zvmsite from posthog import Posthog - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +global start_time +start_time = time() + +""" +Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually +""" + callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') @@ -28,245 +35,119 @@ client_id = os.environ.get('CLIENT_ID', 'api-script') client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) api_timeout = int(os.environ.get('API_TIMEOUT', 5)) -LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() +LOGLEVEL = os.environ.get('LOGLEVEL', 'DEBUG').upper() +DISABLE_STATS = os.environ.get('DISABLE_STATS', 'FALSE').upper() version = str(VERSION) vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') -# Get the hostname of the machine -container_id = str(socket.gethostname()) -#set log line format including container_id -log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") -log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) -log_handler.setFormatter(log_formatter) - -log = logging.getLogger("Node-Exporter") -log.setLevel(LOGLEVEL) -log.addHandler(log_handler) -log.info(f"Zerto-Node-Exporter - Version {version}") -log.info(f"Log Level: {LOGLEVEL}") -log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) - -# Global Variables -token = "" -siteId = "NotSet" -siteName = "NotSet" -siteZvmVersion = "" -siteVcVersion = "" -siteZvmMajorVersion = "" -siteZvmMinorVersion = "" -siteZvmUpdateVersion = "" -siteZvmPatchVersion = "" -lastStats = CaseInsensitiveDict() - -# Check if vCenter is set, if not disable VRA metrics -is_vcenter_set = True -if vcenter_host == "vcenter.local": - log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") - is_vcenter_set = False -log.debug("vCenter data collection is enabled") -vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") - -# Authentication Thread which handles authentication and token refresh for ZVM API -def ZvmAuthHandler(): - log.debug("ZVMAuthHandler Thread Started") - expiresIn = 0 - global token - global siteId - global siteName - retries = 0 - while True: - if expiresIn < 30: - h = CaseInsensitiveDict() - h["Content-Type"] = "application/x-www-form-urlencoded" - - d = CaseInsensitiveDict() - d["client_id"] = client_id - d["client_secret"] = client_secret - d["grant_type"] = "client_credentials" - - uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token" - delay = 0 - - try: - response = requests.post(url=uri, data=d, headers=h, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: - log.error("Authentication response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - token = str(responseJSON.get('access_token')) - expiresIn = int(responseJSON.get('expires_in')) - log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds") - - if response.status_code != 200: - log.error("Authentication request failed with status code " + str(response.status_code)) - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - # Get Site ID and Name - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite" - delay = 0 - try: - log.debug("Getting Site ID and Name") - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - log.debug(responseJSON) - if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: - log.error("LocalSite API response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - else: - siteId = str(responseJSON.get('SiteIdentifier')) - siteName = str(responseJSON.get('SiteName')) - siteZvmVersion = str(responseJSON.get('Version')) - siteVcVersion = str(responseJSON.get('SiteTypeVersion')) - - # Break out ZVM version strings - siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") - siteZvmUpdateVersion = siteZvmUpdateVersion[0] - if (len(siteZvmUpdateVersion) > 1): - siteZvmPatchVersion = siteZvmUpdateVersion[1] - else: - siteZvmPatchVersion = "0" - log.info("Site ID: " + siteId + " Site Name: " + siteName) - - expiresIn -= 10 + delay - log.debug("Token Expires in " + str(expiresIn) + " seconds") - sleep(10) - -''' # Thread which gets VM level encryption statistics from ZVM API -def GetStatsFunc(): +def GetStatsFunc(zvm_instance): tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging dbvm = Query() dbvpg = Query() + zvm = zvm_instance while (True) : - global token global siteId global siteName - if (token != ""): - log.info("Got Auth Token!") - log.debug("token: " + str(token)) + if (zvm.is_authenticated()): log.debug("Stats Collector Loop Running") metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token ## Statistics API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" - statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - statsapi_json = statsapi.json() - #log.debug(statsapi_json) + statsapi_json = None + statsapi_json = zvm.vms_statistics() + log.debug(statsapi_json) + vms_encryption_metrics = zvm.encryptiondetection_metrics_vms() - for vm in statsapi_json: - oldvmdata = dict() + if statsapi_json is not None: + for vm in statsapi_json: + vmsiteinfo = zvm.vm(vmidentifier=vm['VmIdentifier'], vpgidentifier=vm['VpgIdentifier']) + if vmsiteinfo['ProtectedSite']['identifier'] == zvm.site_id: + log.debug(f"VM is protected at this site - {vm['VmIdentifier']}") + oldvmdata = dict() + if 'EncryptionMetrics' not in vm: + vm['EncryptionMetrics'] = {} + vm['VmName'] = None - CurrentIops = 0 - CurrentWriteCounterInMBs = 0 - CurrentSyncCounterInMBs = 0 - CurrentNetworkTrafficCounterInMBs = 0 - CurrentEncryptedLBs = 0 - CurrentUnencryptedLBs = 0 - CurrentTotalLBs = 0 - CurrentPercentEncrypted = 0 - VMName = "NA" + CurrentIops = 0 + CurrentWriteCounterInMBs = 0 + CurrentSyncCounterInMBs = 0 + CurrentNetworkTrafficCounterInMBs = 0 + CurrentEncryptedLBs = 0 + CurrentUnencryptedLBs = 0 + CurrentTotalLBs = 0 + CurrentPercentEncrypted = 0 + CurrentTrendChangeLevel = 0 + VMName = "NA" - oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + for vmem in vms_encryption_metrics: + if vmem['Link']['identifier'] == vm['VmIdentifier']: + log.debug(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}") + #print(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}") + vm['EncryptionMetrics']['EncryptedData'] = vmem['EncryptionMetrics']['EncryptedData'] + vm['EncryptionMetrics']['NonEncryptedData'] = vmem['EncryptionMetrics']['NonEncryptedData'] + vm['EncryptionMetrics']['TrendChangeLevel'] = vmem['EncryptionMetrics']['TrendChangeLevel'] + vm['VmName'] = vmem['Link']['name'] - log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) - if (oldvmdata): - log.info(vm['VmIdentifier'] + " Record Found, Updating DB") - log.debug(oldvmdata[0]) - log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) + log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) + oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + if (oldvmdata): + log.info(vm['VmIdentifier'] + " Record Found, Updating DB") + log.debug(oldvmdata[0]) + log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) - log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") - VMName = oldvmdata[0]['VmName'] - log.debug("Current VM " + str(VMName)) - CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) - log.debug("CurrentIops " + str(CurrentIops)) - CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) - log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) - CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) - log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) - CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) - log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) - CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) - log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) - CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) - log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) - if CurrentTotalLBs != 0: - CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") + VMName = oldvmdata[0]['VmName'] + log.debug("Current VM " + str(VMName)) + CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) + log.debug("CurrentIops " + str(CurrentIops)) + CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) + log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) + CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) + log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) + CurrentWriteCounterInMBs = abs(vm['WriteCounterInMBs'] - oldvmdata[0]['WriteCounterInMBs']) + log.debug("CurrentWriteCounterInMBs " + str(CurrentWriteCounterInMBs)) + CurrentEncryptedLBs = abs(vm['EncryptionMetrics']['EncryptedData'] - oldvmdata[0]['EncryptionMetrics']['EncryptedData']) + log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) + CurrentUnencryptedLBs = abs(vm['EncryptionMetrics']['NonEncryptedData'] - oldvmdata[0]['EncryptionMetrics']['NonEncryptedData']) + log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) + CurrentTrendChangeLevel = abs(vm['EncryptionMetrics']['TrendChangeLevel'] - oldvmdata[0]['EncryptionMetrics']['TrendChangeLevel']) + log.debug("CurrentTrendChangeLevel " + str(CurrentTrendChangeLevel)) + CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) + log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) + if CurrentTotalLBs != 0: + CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + else: + CurrentPercentEncrypted = 0 + log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + else: + log.info(f"{vm['VmIdentifier']} - {vm['VmName']} - No Record Found, Inserting into DB") + #insert original VM record to tempdb + log.debug(tempdb.insert(vm)) + + # Store Calculated Metrics + metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops + metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs + metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs + metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs + #metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs + #metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs + #metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs + #metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted else: - CurrentPercentEncrypted = 0 - log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") + #print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") + else: + log.debug("No VMS in Stats API") - else: - log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") - #insert original VM record to tempdb - log.debug(tempdb.insert(vm)) - - # update database with VM name, for easier display in Grafana Legends - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] - try: - vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vapi_json = vapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - else: - log.debug("vapi_json: " + str(vapi_json)) - tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) - log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) - VMName = vapi_json['VmName'] - - # Store Calculated Metrics - metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops - metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs - metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs - metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs - metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs - metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs - metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs - metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus file_object = open('statsmetrics', 'w') @@ -288,134 +169,112 @@ def GetStatsFunc(): else: log.debug("Waiting 1 second for Auth Token") sleep(1) -''' # Function which retrieves stats from various ZVM APIs and stores them in a metrics file -def GetDataFunc(): +def GetDataFunc(zvm_instance): tempdb = TinyDB(storage=MemoryStorage) dbvm = Query() + zvm = zvm_instance while (True) : - global token global siteId global siteName - if (token != ""): - log.debug("Got Auth Token!") - log.debug("token: " + str(token)) + if (zvm.is_authenticated()): log.info("Data Collector Loop Running") - metricsDictionary = {} - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - ### VPGs API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - vpg_json = service.json() - #log.debug(vpg_json) - for vpg in vpg_json : - metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] - metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] - metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] - metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] - metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] - metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] - metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] - metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] - metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] - metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] - metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] - metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] - metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] - metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] + vpg_json = None + vpg_json = zvm.vpgs() + if(vpg_json is not None): + log.debug("Got VPG JSON") + for vpg in vpg_json : + metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] + metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] + metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] + metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] + metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] + metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] + metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] + metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] + metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] + metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] + metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] + metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] + metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] + metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] + else: + log.debug("No VPGs Found") ### Datastores APIs - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - ds_json = service.json() - #log.debug(ds_json) - for ds in ds_json : - - log.debug(f"Processing {ds['DatastoreName']}") - - metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] - metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] - metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] - metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] - metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] - metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] - metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] + ds_json = None + ds_json = zvm.datastores() + if(ds_json is not None): + log.debug("Got Datastores API") + for ds in ds_json : + log.debug(f"Processing {ds['DatastoreName']}") + metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] + metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] + metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] + metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] + metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] + metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] + metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] + else: + log.debug("No Datastores Found") ## VMs API log.debug("Getting VMs API") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + scratch_vols = None + scratch_vols = zvm.vms() + if(scratch_vols is not None): + log.debug("Got VMs API") + for vm in scratch_vols: + log.debug("Processing VM: " + str(vm['VmName'])) + log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) - vmapi_json = {} - try: - vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vmapi_json = vmapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return + if siteId == vm['ProtectedSite']['identifier']: + log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - log.debug("Got VMs API") - log.debug(vmapi_json) - for vm in vmapi_json : - log.debug("Processing VM: " + str(vm['VmName'])) - log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) + if not isinstance(vm["ActualRPO"], int): + vm["ActualRPO"] = -1 + metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] + metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] + metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] + metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] + metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] + metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] + metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] + metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] + metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] + metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] + metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] + log.debug("Processed VM: " + str(vm['VmName'])) - if siteId == vm['ProtectedSite']['identifier']: - log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - - if not isinstance(vm["ActualRPO"], int): - vm["ActualRPO"] = -1 - metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] - metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] - metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] - metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] - metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] - metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] - metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] - metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] - metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] - metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] - metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] - log.debug("Processed VM: " + str(vm['VmName'])) - - else: - log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") + else: + log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") + else: + log.debug("No VMs Found") ## Volumes API for Scratch Volumes log.debug("Getting Scratch Volumes") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch" + scratch_vols = None + scratch_vols = zvm.volumes(volumetype="scratch") - volapi_json = {} - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Scratch Volumes API") - if(bool(volapi_json)): - for volume in volapi_json : + if(scratch_vols is not None): + log.debug("Got Scratch Volumes API") + for volume in scratch_vols: #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" @@ -426,24 +285,17 @@ def GetDataFunc(): percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100) percentage_used = round(percentage_used, 1) #metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used + else: + log.debug("No Scratch Volumes Found") ## Volumes API for Journal Volumes log.debug("Getting Journal Volumes") + journal_vols = None + journal_vols = zvm.volumes(volumetype="journal") - volapi_json = {} - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal" - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Journal Volumes API") - if(bool(volapi_json)): + if(journal_vols is not None): log.debug("Journal Volumes Exist") - for volume in volapi_json : + for volume in journal_vols : log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...") #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. @@ -464,6 +316,8 @@ def GetDataFunc(): metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1 else: metricsDictionary[metrickey] = 1 + else: + log.debug("No Journal Volumes Exist") ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus log.debug("Writing metrics to file") @@ -491,101 +345,89 @@ def GetDataFunc(): sleep(1) # get VRA CPU and memory usage from vCenter Server -def GetVraMetrics(): - # set up API endpoint and headers - log.debug("GetVraCpuMemory() called") - metricsDictionary = {} - while True: - vra_names = [] - vras = [] - global token - global siteId - global siteName +def GetVraMetrics(zvm_instance): + log.debug("GetVraMetrics thread started") + try: - log.debug("Checking Token in VRA CPU MEM Collector") - if (token != ""): - log.debug("Auth Token Valid!") - log.debug("token: " + str(token)) - log.info("VRA CPU MEM Collector Running") + metricsDictionary = {} + zvm = zvm_instance + while True: + vra_names = [] + vras = [] + global siteId + global siteName - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - + log.debug("Checking Token in VRA CPU MEM Collector") + if (zvm.is_authenticated()): + log.info("VRA CPU MEM Collector Running") - ### VRA API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras" - - # make API call to get list of VRAs - try: - response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - except Exception as e: - log.error(f"Error connecting to {uri}: {e}") - return - else: - log.debug("Response from GET /v1/vras: %s", response.text) - # parse JSON response and get the name of each VRA - - vras = response.json() - - log.debug("VRA names: %s", vras) - log.debug(type(vras)) - for vra in vras : - # Gather other VRA Metrics from Zerto API into Metrics Diectionary - metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] - metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] - metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] - metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] - metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] - metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] - metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] - metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] - metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - - - log.debug("VRA Name: %s", vra['VraName']) - log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") - - # get the CPU and memory usage for each VRA - if is_vcenter_set: - log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") - try: - log.debug("Trying to get stats from vc module") - vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + ### VRA API + vras_json = None + vras_json = zvm.vras() + log.debug(vras_json) - # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vradata[0] - memory_usage_mb = vradata[1] + if (vras_json is not None): + log.debug("VRA names: %s", vras_json) + log.debug(type(vras)) + for vra in vras_json : + # Gather other VRA Metrics from Zerto API into Metrics Diectionary + metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] + metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] + metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] + metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] + metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] + metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] + metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] + metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] + metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - # print the CPU and memory usage for the VM - log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") - metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz - metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb - except: - log.info(f"No VM found with name {vra['VraName']}") - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('vrametrics', 'w') - txt_object = open('vrametrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() + log.debug("VRA Name: %s", vra['VraName']) + log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user}") - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") - sleep(scrape_speed * 2) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) + # get the CPU and memory usage for each VRA + if is_vcenter_set: + log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") + try: + log.debug("Trying to get stats from vc module") + vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + + # get the CPU usage and memory usage for the VM + cpu_usage_mhz = vradata[0] + memory_usage_mb = vradata[1] + + # print the CPU and memory usage for the VM + log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz + metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb + except: + log.info(f"No VM found with name {vra['VraName']}") + else: + log.debug("No VRAs Found") + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + file_object = open('vrametrics', 'w') + txt_object = open('vrametrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + + file_object.close() + txt_object.close() + + # This function will get data every 10 seconds + log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") + sleep(scrape_speed * 2) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) + except Exception as e: + log.error(f"Error in GetVraMetrics: {e}") # function which monitors the threads and restarts them if they die def ThreadProbe(): @@ -594,28 +436,27 @@ def ThreadProbe(): log.debug("Thread Probe Started") metricsDictionary = {} - log.debug("Is Auth Thread Alive") - if auth_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is Data Thread Alive") + uptime = round((time() - start_time) / 60, 1) + metricsDictionary["exporter_uptime{ExporterInstance=\"" + container_id + "\"}"] = uptime if data_thread.is_alive(): + log.debug("Data Thread Is Alive") metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 else: + log.debug("Data Thread Is NOT Alive") metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - #log.debug("Is Stats Thread Alive") - #if stats_thread.is_alive(): - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - #else: - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + if stats_thread.is_alive(): + log.debug("Stats Thread Is Alive") + metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + else: + log.debug("Stats Thread Is NOT Alive") + metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - log.debug("Is VRA Metrics Thread Alive") if vra_metrics_thread.is_alive(): + log.debug("VRA Metrics Thread Is Alive") metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 else: + log.debug("VRA Metrics Thread Is NOT Alive") metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 log.debug("Writing Probe data to files") @@ -638,7 +479,6 @@ def ThreadProbe(): log.debug("Probe Thread Going to Sleep") sleep(30) - #----------------run http server on port 9999----------------- def WebServer(): log.info("Web Server Started") @@ -651,17 +491,75 @@ def WebServer(): httpd.serve_forever() def start_thread(target_func): - # start a new thread + log.debug(f"Starting thread for {target_func.__name__}") thread = threading.Thread(target=target_func) + thread.daemon = True thread.start() - # return the thread object + log.debug(f"Thread {target_func.__name__} started") return thread -# start the threads -auth_thread = start_thread(ZvmAuthHandler) -data_thread = start_thread(GetDataFunc) -#stats_thread = start_thread(GetStatsFunc()) -vra_metrics_thread = start_thread(GetVraMetrics) +""" +Main Program Logic +""" + +# Get the hostname of the machine +container_id = str(socket.gethostname()) + +#set log line format including container_id +log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") +log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) +log_handler.setFormatter(log_formatter) +log = logging.getLogger("Node-Exporter") +log.setLevel(LOGLEVEL) +log.addHandler(log_handler) +log.info(f"Zerto-Node-Exporter - Version {version}") +log.info(f"Log Level: {LOGLEVEL}") +log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) + +# Initialize zvmsite instance +zvm_instance = zvmsite( + host=zvm_url, + port=zvm_port, + client_id=client_id, + client_secret=client_secret, + grant_type="client_credentials", + loglevel=LOGLEVEL, + logger=log, + stats=DISABLE_STATS +) +# Start the zvmsite authentication thread +zvm_instance.connect() + +""" +Global Variables used by the program +""" +local_site_info = None +siteId = None +siteName = None + +while(siteId is None): + if zvm_instance.is_authenticated(): + sleep(2) + log.debug("Trying Set Global Vars") + siteId = zvm_instance.site_id + siteName = zvm_instance.site_name + +lastStats = CaseInsensitiveDict() + +# Check if vCenter is set, if not disable VRA metrics +is_vcenter_set = True +if vcenter_host == "vcenter.local": + log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") + is_vcenter_set = False +log.debug("vCenter data collection is enabled") +vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug", logger=log) + +# Starting threads +vra_metrics_thread = start_thread(lambda: GetVraMetrics(zvm_instance)) +data_thread = start_thread(lambda: GetDataFunc(zvm_instance)) +stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) +log.debug("Starting VRA Metrics") + webserver_thread = start_thread(WebServer) probe_thread = start_thread(ThreadProbe) @@ -673,22 +571,18 @@ while True: # restart the thread log.error("Probe Thread Died - Restarting") probe_thread = start_thread(ThreadProbe) - if not auth_thread.is_alive(): - # restart the thread - log.error("Authentication Thread Died - Restarting") - auth_thread = start_thread(ZvmAuthHandler) if not data_thread.is_alive(): # restart the thread log.error("Data Thread Died - Restarting") - data_thread = start_thread(GetDataFunc) - #if not stats_thread.is_alive(): - # # restart the thread - # log.error("Stats Thread Died - Restarting") - # stats_thread = start_thread(GetStatsFunc()) + data_thread = start_thread(GetDataFunc(zvm_instance)) + if not stats_thread.is_alive(): + # restart the thread + log.error("Stats Thread Died - Restarting") + stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) if not vra_metrics_thread.is_alive(): # restart the thread log.error("VRA Metrics Thread Died - Restarting") - vra_metrics_thread = start_thread(GetVraMetrics) + vra_metrics_thread = start_thread(GetVraMetrics(zvm_instance)) if not webserver_thread.is_alive(): # restart the thread log.error("Webserver Thread Died - Restarting") diff --git a/app/vmware/__init__.py b/app/vmware/__init__.py index a5312ec..97bca9c 100644 --- a/app/vmware/__init__.py +++ b/app/vmware/__init__.py @@ -1,4 +1,4 @@ -print("Initializing zvma10 package...") +print("Initializing vcenter package...") #from .zvma import zvm from .vcenter import vcsite \ No newline at end of file diff --git a/app/vmware/__pycache__/__init__.cpython-310.pyc b/app/vmware/__pycache__/__init__.cpython-310.pyc index ab8f405..c2023c5 100644 Binary files a/app/vmware/__pycache__/__init__.cpython-310.pyc and b/app/vmware/__pycache__/__init__.cpython-310.pyc differ diff --git a/app/vmware/__pycache__/vcenter.cpython-310.pyc b/app/vmware/__pycache__/vcenter.cpython-310.pyc index 10f0762..736e798 100644 Binary files a/app/vmware/__pycache__/vcenter.cpython-310.pyc and b/app/vmware/__pycache__/vcenter.cpython-310.pyc differ diff --git a/app/vmware/vcenter.py b/app/vmware/vcenter.py index 6bbe148..92a3f50 100644 --- a/app/vmware/vcenter.py +++ b/app/vmware/vcenter.py @@ -4,10 +4,11 @@ from pyVmomi import vim, vmodl import ssl import datetime import logging +import socket from logging.handlers import RotatingFileHandler class vcsite: - def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO"): + def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO", logger=None): self.host = host self.port = port self.username = username @@ -16,21 +17,26 @@ class vcsite: self.version = None self.__conn__ = None self.LOGLEVEL = loglevel.upper() + self.log = None - #set log line format including container_id - log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") - log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5) - log_handler.setFormatter(log_formatter) - self.log = logging.getLogger("Node-Exporter") - self.log.setLevel(self.LOGLEVEL) - self.log.addHandler(log_handler) + if logger is None: + #set log line format including container_id + container_id = str(socket.gethostname()) + log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") + log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) + log_handler.setFormatter(log_formatter) + self.log = logging.getLogger("vCenter Module") + self.log.setLevel(self.LOGLEVEL) + self.log.addHandler(log_handler) + else: + self.log = logger def connect(self): self.log.info(f"Log Level set to {self.LOGLEVEL}") if self.__conn__ is None: context = ssl.create_default_context() if not self.verify_ssl: - print("dont verify SSL") + log.debug("dont verify SSL") # Create an SSL context without certificate verification context.check_hostname = False context.verify_mode = ssl.CERT_NONE diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py index 1367eb7..a199b00 100644 --- a/app/zvma10/zvma.py +++ b/app/zvma10/zvma.py @@ -12,6 +12,8 @@ from urllib3.exceptions import InsecureRequestWarning from urllib.parse import urlencode from urllib.parse import urlparse from time import sleep +from datetime import datetime +from dateutil import parser from typing import List, Dict, Tuple, Union, Any, Optional from requests.structures import CaseInsensitiveDict from logging.handlers import RotatingFileHandler @@ -21,7 +23,7 @@ from requests import Request, Session from .version import VERSION class zvmsite: - def __init__(self, host, username=None, password=None, port: int = 443, verify_ssl: bool = False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug", stats: bool = True): + def __init__(self, host, username=None, password=None, port: int = 443, verify_ssl: bool = False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug", logger=None, stats: bool = True) -> None: self.stats = stats self.host = host self.port = port @@ -61,7 +63,11 @@ class zvmsite: self._running = False self.LOGLEVEL = loglevel.upper() - self.setup_logging() + if logger is None: + self.setup_logging() + else: + self.log = logger + atexit.register(self.disconnect) self._running = True @@ -74,7 +80,7 @@ class zvmsite: self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded') self.log.debug("Sent PostHog Hook") - def __authhandler__(self): + def __authhandler__(self) -> None: self.log.info(f"Log Level set to {self.LOGLEVEL}") if not self.__connected__: context = ssl.create_default_context() @@ -110,6 +116,10 @@ class zvmsite: self.expiresIn = int(response['expires_in']) self.log.info("Authentication successful") self.__connected__ = True + local_site_info = self.local_site() + self.site_id = local_site_info['SiteIdentifier'] + self.site_name = local_site_info['SiteName'] + else: self.log.error("Authentication failed") sleep(2 ** retries) @@ -131,7 +141,7 @@ class zvmsite: log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) log_handler.setFormatter(log_formatter) - self.log = logging.getLogger("Node-Exporter") + self.log = logging.getLogger("ZVM10 Module") self.log.setLevel(self.LOGLEVEL) self.log.addHandler(log_handler) @@ -322,6 +332,23 @@ class zvmsite: return False # Return False if the try block didn't execute successfully + def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, + entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]: + + params = { + 'startdate': startdate, + 'enddate': enddate, + 'vpgid': vpgid, + 'zorgidentifier': zorgidentifier, + 'level': level, + 'entity': entity, + 'helpidentifier': helpidentifier, + 'isdismissed': isdismissed + } + + uri = self.construct_url("v1/alerts", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + def alert_levels(self) -> List[str]: params = { @@ -345,25 +372,8 @@ class zvmsite: uri = self.construct_url(f"v1/alerts/helpidentifiers", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, - entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]: - - params = { - 'startdate': startdate, - 'enddate': enddate, - 'vpgid': vpgid, - 'zorgidentifier': zorgidentifier, - 'level': level, - 'entity': entity, - 'helpidentifier': helpidentifier, - 'isdismissed': isdismissed - } - - uri = self.construct_url("v1/alerts", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def datastore(self, datastoreidentifier=None): + + def datastore(self, datastoreidentifier=None) -> Dict[str, Any]: if datastoreidentifier is None: self.log.error("Datastore identifier is required for get_datastore function.") @@ -382,6 +392,59 @@ class zvmsite: uri = self.construct_url("v1/datastores", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def datetime_local(self) -> datetime: + params = {} + uri = self.construct_url(f"v1/serverDateTime/serverDateTimeLocal", {}) + response = self.make_api_request("GET", uri, headers=self.apiheader) + + if response is not None: + # Extract the datetime string from the JSON response + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise ValueError(error_message) + + def datetime_utc(self,) -> datetime: + params = {} + uri = self.construct_url(f"v1/serverDateTime/serverDateTimeUtc", params) + response = self.make_api_request("GET", uri, headers=self.apiheader) + + if response is not None: + # Extract the datetime string from the JSON response + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise ValueError(error_message) + + def datetime_check(self, dt_str: str) -> datetime: + try: + # Try to parse the string into a datetime object + dt = parser.isoparse(dt_str) + except ValueError: + # If parsing fails, raise an error + raise ValueError("The 'dt_str' parameter must be a valid datetime string.") + + # Format the datetime object for the API call + formatted_datetime = dt.isoformat() + + params = {'datetime': formatted_datetime} + + # Construct the URL with the datetime argument + uri = self.construct_url(f"v1/serverDateTime/dateTimeArgument", params) + + # Make the API request + response = self.make_api_request("GET", uri, headers=self.apiheader) + + # Check if the response is not None and parse the datetime string + if response is not None: + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise requests.exceptions.HTTPError(error_message) def encryptiondetection_enable(self): @@ -450,7 +513,7 @@ class zvmsite: uri = self.construct_url("v1/encryptionDetection/suspected/vpgs", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def event(self, eventidentifier=None): + def event(self, eventidentifier=None) -> Dict[str, Any]: if eventidentifier is None: self.log.error("Event identifier is required for get event function.") @@ -461,30 +524,6 @@ class zvmsite: uri = self.construct_url(f"v1/events/{eventidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_types(self): - - params = { - } - - uri = self.construct_url(f"v1/events/types", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_entities(self): - - params = { - } - - uri = self.construct_url(f"v1/events/entities", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_categories(self): - - params = { - } - - uri = self.construct_url(f"v1/events/categories", params) - return self.make_api_request("GET", uri, headers=self.apiheader) def events(self, startdate=None, enddate=None, vpgid=None, sitename=None, zorgidentifier=None, eventtype=None, entitytype=None, category=None, username=None, alertidentifier=None) -> List[Dict[str, Any]]: @@ -504,8 +543,74 @@ class zvmsite: uri = self.construct_url("v1/events", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_types(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/types", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_entities(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/entities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_categories(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/categories", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def license(self) -> Dict[str, Any]: + + params = { + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("GET", uri, headers=self.apiheader) - def local_site(self): + def license_delete(self) -> bool: + params = {} + uri = self.construct_url(f"v1/license", params) + + try: + response = self.make_api_request("DELETE", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to delete license: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending license delete request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def license_apply(self, license=None): + if license is None: + self.log.error("A license key is required for apply license function.") + raise ValueError("License key is required.") + + params = { + } + + license = { + "licenseKey": license + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) + + def local_site(self) -> Dict[str, Any]: params = { } @@ -521,15 +626,26 @@ class zvmsite: uri = self.construct_url(f"v1/localsite/pairingstatuses", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def local_site_send_billing(self): - - params = { - } - + def local_site_send_billing(self) -> bool: + params = {} uri = self.construct_url(f"v1/localsite/settings/sendusage", params) - return self.make_api_request("POST", uri, headers=self.apiheader) - - def local_site_banner(self): + + try: + response = self.make_api_request("POST", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to send billing information: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending billing information request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def local_site_banner(self) -> Dict[str, Any]: params = { } @@ -549,40 +665,7 @@ class zvmsite: # uri is spelled incorrectly because it is also spelled incorrectly in zerto uri = self.construct_url(f"v1/localsite/settings/logingbanner", params) return self.make_api_request("PUT", uri, json_data=data, headers=self.apiheader) - - def license(self): - params = { - } - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def license_delete(self): - - params = { - } - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("DELETE", uri, headers=self.apiheader) - - def license_apply(self, license=None): - - if license is None: - self.log.error("A license key is required for apply license function.") - raise ValueError("License key is required.") - - params = { - } - - license = { - "licenseKey": license - } - - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) - def peer_sites(self) -> List[Dict[str, Any]]: params = { @@ -591,7 +674,7 @@ class zvmsite: uri = self.construct_url(f"v1/peersites", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def peer_site(self, siteidentifier=None): + def peer_site(self, siteidentifier=None) -> Dict[str, Any]: if siteidentifier is None: self.log.error("Site identifier is required for get site function.") raise ValueError("Site identifier is required.") @@ -602,7 +685,7 @@ class zvmsite: uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def peer_sites_pairing_statues(self): + def peer_sites_pairing_statues(self) -> List[str]: params = { } @@ -644,12 +727,83 @@ class zvmsite: uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) return self.make_api_request("DELETE", uri, json=data, headers=self.apiheader) - def peer_site_pairing_token(self): + def peer_site_pairing_token(self) -> Dict[str, Any]: params = {} uri = self.construct_url(f"v1/peersites/generatetoken", params) return self.make_api_request("POST", uri, headers=self.apiheader) + def recovery_reports(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, vpgname=None, recoverytype=None, state=None) -> List[Dict[str, Any]]: + + params = { + 'starttime': starttime, + 'endtime': endtime, + 'pagenumber': pagenumber, + 'pagesize': pagesize, + 'vpgname': vpgname, + 'recoverytype': recoverytype, + 'state': state + } + + uri = self.construct_url("v1/reports/recovery", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def recovery_report(self, recoveryoperationidentifier=None) -> Dict[str, Any]: + + if recoveryoperationidentifier is None: + self.log.error("RecoveryOperationIdentifier is required for function.") + raise ValueError("RecoveryOperationIdentifier is required.") + + params = {} + + uri = self.construct_url(f"v1/reports/recovery/{recoveryoperationidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def resources_report(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, zorgname=None, vpgname=None, vmname=None, + protectedsitename=None, protectedclustername=None, protectedhostname=None, protectedorgvdc=None, protectedvcdorg=None, recoverysitename=None, + recoveryclustername=None, recoveryhostname=None, recoveryorgvdc=None, recoveryvcdorg=None) -> List[Dict[str, Any]]: + + params = { + 'starttime': starttime, + 'endtime': endtime, + 'pagenumber': pagenumber, + 'pagesize': pagesize, + 'vpgname': vpgname, + 'vmname': vmname, + 'protectedsitename': protectedsitename, + 'protectedclustername': protectedclustername, + 'protectedhostname': protectedhostname, + 'protectedorgvdc': protectedorgvdc, + 'protectedvcdorg': protectedvcdorg, + 'recoverysitename': recoverysitename, + 'recoveryclustername': recoveryclustername, + 'recoveryhostname': recoveryhostname, + 'recoveryorgvdc': recoveryorgvdc, + 'recoveryvcdorg': recoveryvcdorg + } + + uri = self.construct_url("v1/reports/resources", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def service_profiles(self) -> List[Dict[str, Any]]: + + params = { + } + + uri = self.construct_url(f"/v1/serviceprofiles", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def service_profile(self, serviceProfileIdentifier=None) -> Dict[str, Any]: + if siteidentifier is None: + self.log.error("Service Profile identifier is required for get site function.") + raise ValueError("Service Profile identifier is required.") + + params = { + } + + uri = self.construct_url(f"/v1/serviceprofiles/{serviceProfileIdentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None) -> List[Dict[str, Any]]: params = { @@ -664,7 +818,7 @@ class zvmsite: uri = self.construct_url("v1/tasks", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def task(self, taskidentifier=None): + def task(self, taskidentifier=None) -> Dict[str, Any]: if taskidentifier is None: self.log.error("Task identifier is required for function.") @@ -675,7 +829,7 @@ class zvmsite: uri = self.construct_url(f"v1/tasks/{taskidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def task_types(self): + def task_types(self) -> List[str]: params = { } @@ -683,56 +837,13 @@ class zvmsite: uri = self.construct_url(f"v1/tasks/types", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def vpg(self, vpgidentifier=None): + def vms_statistics(self) -> List[Dict[str, Any]]: - if vpgidentifier is None: - self.log.error("Vpg identifier is required for get_vpg function.") - raise ValueError("VM identifier is required.") - - params = { - } + params = { } - uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params) + uri = self.construct_url("v1/statistics/vms", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, - recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, - zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]: - - params = { - 'vpgid': vpgid, - 'vpgname': vpgname, - 'vpgstatus': vpgstatus, - 'vpgsubstatus': vpgsubstatus, - 'protectedsitetype': protectedsitetype, - 'recoverysitetype': recoverysitetype, - 'protectedsiteidentifier': protectedsiteidentifier, - 'recoverysiteidentifier': recoverysiteidentifier, - 'zorgidentifier': zorgidentifier, - 'priority': priority, - 'serviceprofileidentifier': serviceprofileidentifier - } - - uri = self.construct_url("v1/vpgs", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True): - if vpgidentifier is None: - self.log.error("VPG identifier is required for delete_vpg function.") - raise ValueError("VPG identifier is required.") - - # URL with vpgidentifier in the path - uri = self.construct_url(f"v1/vpgs/{vpgidentifier}") - - # Data to be sent in the request body - data = { - "keepRecoveryVolumes": keeprecoveryvolumes, - "force": force - } - - # Make the POST request - return self.make_api_request("POST", uri, data=data, headers=self.apiheader) - + def vms(self, vmidentifier=None, vmname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None) -> List[Dict[str, Any]]: @@ -773,8 +884,8 @@ class zvmsite: def vm_pointintime(self, vmidentifier=None, vpgidentifier=None, includebackupvms: bool = None, includemountedvms: bool = None): if vmidentifier is None: - self.log.error("VM identifier is required for get_vm function.") - raise ValueError("VM identifier is required.") + self.log.error("VM identifier is required for vm_pointintime function.") + raise ValueError("VM identifier is required for vm_pointintime.") params = { 'vpgidentifier': vpgidentifier, @@ -783,8 +894,36 @@ class zvmsite: } uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsintime", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - + stats = self.make_api_request("GET", uri, headers=self.apiheader) + + if isinstance(stats, list) and not stats: + self.log.error("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.") + raise ValueError("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.") + elif stats is None: + self.log.error("VM not found, or vpgidentifier must be specified") + raise ValueError("VM not found, or vpgidentifier must be specified") + else: + return stats + + def vm_pointintime_stats(self, vmidentifier=None, vpgidentifier=None): + + if vmidentifier is None: + self.log.error("VM identifier is required for get_vm function.") + raise ValueError("VM identifier is required.") + + params = { + 'vpgidentifier': vpgidentifier + } + + uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsInTime/stats", params) + stats = self.make_api_request("GET", uri, headers=self.apiheader) + + if stats is None: + self.log.error("VM not found, or vpgidentifier must be specified") + raise ValueError("VM not found, or vpgidentifier must be specified") + else: + return stats + def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None) -> List[Dict[str, Any]]: if volumetype: valid_volumetypes = ["scratch", "journal", "recovery", "protected", "appliance"] @@ -805,8 +944,238 @@ class zvmsite: uri = self.construct_url("v1/volumes", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for get_vpg function.") + raise ValueError("VM identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_checkpoints(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_take_checkpoint(self, vpgidentifier=None, checkpointname=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + # Construct the JSON payload + json_payload = {"checkpointname": "Checkpoint By Python ZVM Module"} + if checkpointname is not None: + json_payload["checkpointname"] = checkpointname + + params = { } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params) + return self.make_api_request("POST", uri, json_data=json_payload, headers=self.apiheader) + + def vpg_checkpoint_stats(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints/stats", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, + recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, + zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]: + + params = { + 'vpgid': vpgid, + 'vpgname': vpgname, + 'vpgstatus': vpgstatus, + 'vpgsubstatus': vpgsubstatus, + 'protectedsitetype': protectedsitetype, + 'recoverysitetype': recoverysitetype, + 'protectedsiteidentifier': protectedsiteidentifier, + 'recoverysiteidentifier': recoverysiteidentifier, + 'zorgidentifier': zorgidentifier, + 'priority': priority, + 'serviceprofileidentifier': serviceprofileidentifier + } + + uri = self.construct_url("v1/vpgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) - def __set_zvm_version__(self): + def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True): + if vpgidentifier is None: + self.log.error("VPG identifier is required for delete_vpg function.") + raise ValueError("VPG identifier is required.") + + # URL with vpgidentifier in the path + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}") + + # Data to be sent in the request body + data = { + "keepRecoveryVolumes": keeprecoveryvolumes, + "force": force + } + + # Make the POST request + return self.make_api_request("POST", uri, data=data, headers=self.apiheader) + + def vpg_retention_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/retentionpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_priorities(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/priorities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_entity_types(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/entitytypes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_statuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/statuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_substatuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/substatuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_failover_shutdown_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/failovershutdownpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_failover_commit_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/failovercommitpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vras(self, vraname=None, status=None, vraversion=None, hostname=None, ipaddress=None, + vragroup=None, datastorename=None, datastoreclustername=None, networkname=None, vraipconfigurationapi=None) -> List[Dict[str, Any]]: + + params = { + 'vraname': vraname, + 'status': status, + 'vraversion': vraversion, + 'hostname': hostname, + 'ipaddress': ipaddress, + 'vragroup': vragroup, + 'datastorename': datastorename, + 'datastoreclustername': datastoreclustername, + 'networkname': networkname, + 'vraipconfigurationapi': vraipconfigurationapi + } + + uri = self.construct_url(f"v1/vras", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra(self, vraidentifier=None) -> Dict[str, Any]: + + if vraidentifier is None: + self.log.error("vraidentifier is required for vra function.") + raise ValueError("vraidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/{vraidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra_upgrade(self, vraidentifier=None) -> Dict[str, Any]: + + if vraidentifier is None: + self.log.error("vraidentifier is required for vra function.") + raise ValueError("vraidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/{vraidentifier}/upgrade", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def vra_statuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vras/statuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra_ipconfigurationtypes(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vras/ipconfigurationtypes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + """ + def vra_cluster_settings(self, clusteridentifier=None) -> Dict[str, Any]: + + if clusteridentifier is None: + self.log.error("clusteridentifier is required for vra function.") + raise ValueError("clusteridentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/clusters/{clusteridentifier}/settings", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + """ + + def zorgs(self) -> Dict[str, Any]: + + params = { + } + + uri = self.construct_url(f"v1/zorgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def zorg(self, zorgidentifier=None) -> Dict[str, Any]: + + if zorgidentifier is None: + self.log.error("zorgidentifier is required for function.") + raise ValueError("zorgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/zorgs/{zorgidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def __set_zvm_version__(self) -> None: uri = self.construct_url("v1/localsite") response = self.make_api_request("GET", uri, headers=self.apiheader) if response: @@ -824,7 +1193,7 @@ class zvmsite: self.zvm_version['patch'] = temp[1] if len(temp) > 1 else "0" self.log.info(f"Site ID: {self.site_id}, Site Name: {self.site_name}, Site Type: {self.site_type}") - def version(self): + def version(self) -> Dict[str, Any]: if self.__connected__ and self._running: if self.zvm_version['full'] is None: self.__set_zvm_version__()