From 48dc44416674fd35f1ae3fb4b2a1d9ecaf60e512 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Tue, 24 Oct 2023 19:52:09 -0400 Subject: [PATCH 01/12] dev updates Not finished, but pushing for use on another machine --- .gitignore | 2 + app/iops.py | 59 +++++++++++ app/python-node-exporter.py | 112 +++++++++----------- app/version.py | 2 +- app/zvma10/__init__.py | 4 + app/zvma10/vcenter.py | 201 ++++++++++++++++++++++++++++++++++++ app/zvma10/zvma.py | 83 +++++++++++++++ app/zvma9_7/GetStatsFunc.py | 126 ++++++++++++++++++++++ app/zvma9_7/__init__.py | 3 + 9 files changed, 528 insertions(+), 64 deletions(-) create mode 100644 app/iops.py create mode 100644 app/zvma10/__init__.py create mode 100644 app/zvma10/vcenter.py create mode 100644 app/zvma10/zvma.py create mode 100644 app/zvma9_7/GetStatsFunc.py create mode 100644 app/zvma9_7/__init__.py diff --git a/.gitignore b/.gitignore index 98ccfd7..86fa5b1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ app/vrametrics app/vrametrics.txt app/__pycache__/* app/logs/* +app/zvma10/__pycache__/* +app/zvma9_7/__pycache__/* \ No newline at end of file diff --git a/app/iops.py b/app/iops.py new file mode 100644 index 0000000..d9d8750 --- /dev/null +++ b/app/iops.py @@ -0,0 +1,59 @@ +from pyVim.connect import SmartConnect, Disconnect +from pyVmomi import vim, vmodl +import ssl + +# Create an SSL context without certificate verification +context = ssl.create_default_context() +context.check_hostname = False +context.verify_mode = ssl.CERT_NONE + +si = SmartConnect(host='192.168.50.50', + user='administrator@vsphere.local', + pwd='Zertodata987!', + sslContext=context +) + +# Find the virtual machine by name +vm_name = 'Squid' +content = si.RetrieveContent() +vm = None + +for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: + if obj.name == vm_name: + vm = obj + break + +if vm is None: + print(f"Virtual machine '{vm_name}' not found") + si.Disconnect() + exit(1) + +# Get performance manager +perf_manager = content.perfManager + +# Define the metric ID for write IOPS (counterId = 6) +metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="") + +# Create a real-time query specification +query_spec = vim.PerformanceManager.QuerySpec( + entity=vm, + metricId=[metric_id], + format="normal", +) + +# Query the performance statistics +result = perf_manager.QueryStats(querySpec=[query_spec]) + +if result: + # Get the latest write IOPS value + write_iops = result[0].value[0].value + print(f"Current write IOPS for {vm_name}: {write_iops}") + +# Disconnect from vCenter Server +Disconnect(si) + + + + + + diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index 8182bdc..383ed5a 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -4,31 +4,32 @@ import socketserver import os import ssl import logging +from logging.handlers import RotatingFileHandler import threading import socket from pyVim.connect import SmartConnect, Disconnect from pyVmomi import vim from time import sleep -from logging.handlers import RotatingFileHandler from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.structures import CaseInsensitiveDict from tinydb import TinyDB, Query from tinydbstorage.storage import MemoryStorage from version import VERSION +from zvma10.vcenter import vcsite requests.packages.urllib3.disable_warnings(InsecureRequestWarning) verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') zvm_port = os.environ.get('ZVM_PORT', '443') client_id = os.environ.get('CLIENT_ID', 'api-script') -client_secret = os.environ.get('CLIENT_SECRET', 'js51tDM8oappYUGRJBhF7bcsedNoHA5j') +client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) api_timeout = int(os.environ.get('API_TIMEOUT', 5)) LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() version = str(VERSION) -vcenter_host = os.environ.get('VCENTER_HOST', 'vcenter.local') +vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') -vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'supersecret') +vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') # Get the hostname of the machine container_id = str(socket.gethostname()) @@ -50,6 +51,12 @@ log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host token = "" siteId = "NotSet" siteName = "NotSet" +siteZvmVersion = "" +siteVcVersion = "" +siteZvmMajorVersion = "" +siteZvmMinorVersion = "" +siteZvmUpdateVersion = "" +siteZvmPatchVersion = "" lastStats = CaseInsensitiveDict() # Check if vCenter is set, if not disable VRA metrics @@ -58,6 +65,7 @@ if vcenter_host == "vcenter.local": log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") is_vcenter_set = False log.debug("vCenter data collection is enabled") +vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") # Authentication Thread which handles authentication and token refresh for ZVM API def ZvmAuthHandler(): @@ -141,14 +149,25 @@ def ZvmAuthHandler(): else: siteId = str(responseJSON.get('SiteIdentifier')) siteName = str(responseJSON.get('SiteName')) + siteZvmVersion = str(responseJSON.get('Version')) + siteVcVersion = str(responseJSON.get('SiteTypeVersion')) + + # Break out ZVM version strings + siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") + siteZvmUpdateVersion = siteZvmUpdateVersion[0] + if (len(siteZvmUpdateVersion) > 1): + siteZvmPatchVersion = siteZvmUpdateVersion[1] + else: + siteZvmPatchVersion = "0" log.info("Site ID: " + siteId + " Site Name: " + siteName) expiresIn -= 10 + delay log.debug("Token Expires in " + str(expiresIn) + " seconds") sleep(10) - +''' # Thread which gets VM level encryption statistics from ZVM API + def GetStatsFunc(): tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging dbvm = Query() @@ -266,6 +285,7 @@ def GetStatsFunc(): else: log.debug("Waiting 1 second for Auth Token") sleep(1) +''' # Function which retrieves stats from various ZVM APIs and stores them in a metrics file def GetDataFunc(): @@ -497,44 +517,17 @@ def GetVraMetrics(): try: response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) except Exception as e: - log.error(f"Error connecting to {endpoint}: {e}") + log.error(f"Error connecting to {uri}: {e}") return else: log.debug("Response from GET /v1/vras: %s", response.text) # parse JSON response and get the name of each VRA - - if is_vcenter_set: - # Disable SSL certificate verification - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - # connect to vCenter Server - si = None - try: - si = SmartConnect(host=vcenter_host, user=vcenter_user, pwd=vcenter_pwd, sslContext=context) - log.debug("Connected to vCenter Server %s", vcenter_host) - except Exception as e: - log.error(f"Error connecting to vCenter Server: {e}") - return - - - # get the root folder of the vCenter Server - content = si.RetrieveContent() - root_folder = content.rootFolder - - # create a view for all VMs on the vCenter Server - view_manager = content.viewManager - vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True) - - - - vras = response.json() + vras = response.json() log.debug("VRA names: %s", vras) log.debug(type(vras)) - for vra in vras : - #vra_names.append(vra['VraName']) - + for vra in vras : # Gather other VRA Metrics from Zerto API into Metrics Diectionary metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] @@ -548,31 +541,25 @@ def GetVraMetrics(): log.debug("VRA Name: %s", vra['VraName']) - + log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") # get the CPU and memory usage for each VRA if is_vcenter_set: - vm = None - for vm_obj in vm_view.view: - if vm_obj.name == vra['VraName']: - vm = vm_obj - break - - if vm is not None: - log.debug("Found VRA VM in vCenter with name %s", vra['VraName']) + log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") + try: + log.debug("Trying to get stats from vc module") + vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage - memory_usage_mb = vm.summary.quickStats.guestMemoryUsage + cpu_usage_mhz = vradata[0] + memory_usage_mb = vradata[1] # print the CPU and memory usage for the VM - log.info(f"VM {vm.name} (name: {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb - else: + except: log.info(f"No VM found with name {vra['VraName']}") - - # Disconnect from vCenter - Disconnect(si) ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus file_object = open('vrametrics', 'w') @@ -591,7 +578,7 @@ def GetVraMetrics(): txt_object.close() # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") + log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") sleep(scrape_speed * 2) else: log.debug("Waiting 1 second for Auth Token") @@ -617,11 +604,11 @@ def ThreadProbe(): else: metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - log.debug("Is Stats Thread Alive") - if stats_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + #log.debug("Is Stats Thread Alive") + #if stats_thread.is_alive(): + # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + #else: + # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 log.debug("Is VRA Metrics Thread Alive") if vra_metrics_thread.is_alive(): @@ -669,10 +656,9 @@ def start_thread(target_func): return thread # start the threads - auth_thread = start_thread(ZvmAuthHandler) data_thread = start_thread(GetDataFunc) -stats_thread = start_thread(GetStatsFunc) +#stats_thread = start_thread(GetStatsFunc()) vra_metrics_thread = start_thread(GetVraMetrics) webserver_thread = start_thread(WebServer) probe_thread = start_thread(ThreadProbe) @@ -693,10 +679,10 @@ while True: # restart the thread log.error("Data Thread Died - Restarting") data_thread = start_thread(GetDataFunc) - if not stats_thread.is_alive(): - # restart the thread - log.error("Stats Thread Died - Restarting") - stats_thread = start_thread(GetStatsFunc) + #if not stats_thread.is_alive(): + # # restart the thread + # log.error("Stats Thread Died - Restarting") + # stats_thread = start_thread(GetStatsFunc()) if not vra_metrics_thread.is_alive(): # restart the thread log.error("VRA Metrics Thread Died - Restarting") diff --git a/app/version.py b/app/version.py index 66ef57d..886aefd 100644 --- a/app/version.py +++ b/app/version.py @@ -1,5 +1,5 @@ # version.py -VERSION = "1.2.0" +VERSION = "1.3.0" def main(): # Put your main program code here diff --git a/app/zvma10/__init__.py b/app/zvma10/__init__.py new file mode 100644 index 0000000..a5312ec --- /dev/null +++ b/app/zvma10/__init__.py @@ -0,0 +1,4 @@ +print("Initializing zvma10 package...") + +#from .zvma import zvm +from .vcenter import vcsite \ No newline at end of file diff --git a/app/zvma10/vcenter.py b/app/zvma10/vcenter.py new file mode 100644 index 0000000..65624a7 --- /dev/null +++ b/app/zvma10/vcenter.py @@ -0,0 +1,201 @@ +# Class for holding variables related to a site. +from pyVim.connect import SmartConnect, Disconnect +from pyVmomi import vim, vmodl +import ssl +import datetime +import logging +from logging.handlers import RotatingFileHandler + +class vcsite: + def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO"): + self.host = host + self.port = port + self.username = username + self.password = password + self.verify_ssl = verify_ssl + self.version = None + self.__conn__ = None + self.LOGLEVEL = loglevel.upper() + + #set log line format including container_id + log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") + log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5) + log_handler.setFormatter(log_formatter) + self.log = logging.getLogger("Node-Exporter") + self.log.setLevel(self.LOGLEVEL) + self.log.addHandler(log_handler) + + def connect(self): + self.log.info(f"Log Level set to {self.LOGLEVEL}") + if self.__conn__ is None: + context = ssl.create_default_context() + if not self.verify_ssl: + print("dont verify SSL") + # Create an SSL context without certificate verification + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + # connect to vCenter Server + si = None + try: + self.__conn__ = SmartConnect(host=self.host, user=self.username, pwd=self.password, sslContext=context) + about_info = self.__conn__.content.about + version = about_info.version + self.version = version + self.log.debug("Connected to vCenter Server %s", self.host) + except Exception as e: + self.log.error(f"Error connecting to vCenter Server: {e}") + + def version(self): + return self.version + + def get_cpu_mem_used(self, vra): + if vra == None: + self.log.debug("Get_cpu_mem_used called with no vm name...returning no data") + return + if self.__conn__ == None: + self.log.debug("Trying to get VRA stats without vCenter connection, trying to connect") + self.connect() + + # get the root folder of the vCenter Server + try: + content = self.__conn__.RetrieveContent() + root_folder = content.rootFolder + except: + self.log.debug("Could not get content from vCenter when trying to get VRA stats") + + # create a view for all VMs on the vCenter Server + view_manager = content.viewManager + vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True) + + vm = None + for vm_obj in vm_view.view: + if str(vm_obj.name) == str(vra): + vm = vm_obj + if vm is not None: + self.log.debug(f"Found VRA VM in vCenter with name {vm.name}") + # get the CPU usage and memory usage for the VM + cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage + memory_usage_mb = vm.summary.quickStats.guestMemoryUsage + + # print the CPU and memory usage for the VM + self.log.info(f"VM {vm.name} has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + return [cpu_usage_mhz, memory_usage_mb] + else: + self.log.debug(f"{vm_obj.name} is not a VRA") + raise ValueError("No VRA Found") + + def get_write_iops(self, vm): + try: + content = self.__conn__.RetrieveContent() + except: + self.log.debug("Could not get content from vCenter when trying to get VRA stats") + + # Find the virtual machine by name + vm_name = str(vm) + vm = None + + for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: + if obj.name == vm_name: + vm = obj + break + + if vm is None: + print(f"Virtual machine '{vm_name}' not found") + return + + # Get performance manager + perf_manager = content.perfManager + + # Define the metric ID for write IOPS (counterId = 6) + metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="") + + # calculate the last 60 seconds + end_time = datetime.datetime.now() + start_time = end_time - datetime.timedelta(seconds=60) + + # Create a query specification for roll-up data + query_spec = vim.PerformanceManager.QuerySpec( + entity=vm, + metricId=[metric_id], + format="normal", + startTime=start_time, + endTime=end_time, + intervalId=20, # Use an appropriate interval for the roll-up data + ) + + + # Query the performance statistics + result = perf_manager.QueryStats(querySpec=[query_spec]) + + if result: + # Get the average write IOPS for the last 60 seconds + average_write_iops = sum(result[0].value[0].value) / len(result[0].value[0].value) + print(f"Average write IOPS for the last 60 seconds for {vm_name}: {average_write_iops}") + return average_write_iops + else: + return None + + def get_average_write_latency(self, vm): + try: + content = self.__conn__.RetrieveContent() + except: + self.log.debug("Could not get content from vCenter when trying to get VM stats") + + # Find the virtual machine by name + vm_name = str(vm) + vm = None + + for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: + if obj.name == vm_name: + vm = obj + break + + if vm is None: + self.log.debug(f"Virtual machine '{vm_name}' not found") + return None + + # Get performance manager + perf_manager = content.perfManager + + # Define the metric ID for write latency (counterId = X) - replace X with the correct counter ID + # You'll need to find the specific counter ID for write latency in your vSphere environment. + # The counter for write latency may vary based on your configuration. + + metric_id = vim.PerformanceManager.MetricId(counterId=10, instance="") # Replace X with the correct counter ID + + end_time = datetime.datetime.now() + start_time = end_time - datetime.timedelta(seconds=60) + + # Create a query specification for roll-up data + query_spec = vim.PerformanceManager.QuerySpec( + entity=vm, + metricId=[metric_id], + format="normal", + startTime=start_time, + endTime=end_time, + intervalId=20, # Use an appropriate interval for the roll-up data + ) + + # Query the performance statistics + result = perf_manager.QueryStats(querySpec=[query_spec]) + + if result: + # Get the average write latency for the last 60 seconds + if result[0].value[0].value: + average_write_latency = sum(result[0].value[0].value) / len(result[0].value[0].value) + self.log.info(f"Average write latency for the last 60 seconds for {vm_name}: {average_write_latency}") + return average_write_latency + + return None + + + def disconnect(self): + if self.__conn__ == None: + self.log.debug(f"vCenter disconnect requested, but not currently connected.") + return + # Disconnect from vCenter + Disconnect(self.__conn__) + self.__conn__ = None + self.version = None + self.log.debug(f"Disconnected from vCenter") \ No newline at end of file diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py new file mode 100644 index 0000000..7e8cb1d --- /dev/null +++ b/app/zvma10/zvma.py @@ -0,0 +1,83 @@ +# Class for holding variables related to a site. + +class site: + def __init__(self): + self.zvm_ip = None + self.zvm_port = 443 + self.zvm_client = None + self.zvm_secret = None + self.zvm_verify_ssl = False + self.zvm_token = None + self.id = None + self.name = None + self.zvm_version = None + self.zvm_version_major = None + self.zvm_version_minor = None + self.zvm_version_update = None + self.zvm_version_patch = None + + self.vc_ip = None + self.vc_port = 443 + self.vc_username = None + self.vc_password = None + self.vc_verify_ssl = False + self.vc_version = None + + + # ZVM Related Set / Get Functions + def set_zvm_ip(self, value): + self.zvm_ip = value + + def set_zvm_port(self, value): + self.zvm_port = value + + def set_zvm_client(self, value): + self.client = value + + def set_zvm_secret(self, value): + self.zvm_secret = value + + def set_zvm_verify_ssl(self, value): + self.zvm_verify_ssl = value + + def set_zvm_token(self, value): + self.set_zvm_token = value + + def set_zvm_id(self, value): + self.id = value + + def set_zvm_name(self, value): + self.zvm_name = value + + def set_zvm_version(self, value): + # Set main zvm version variable + self.zvm_version = value + + # Break out ZVM version string into Major, Minor, Update, Patch variables + self.zvm_version_major, self.zvm_version_minor, temp = self.zvm_version.split(".") + self.zvm_version_update = temp[0] + if (len(temp) > 1): + self.zvm_version_patch = temp[1] + else: + self.zvm_version_patch = "0" + + def set_zvm_token(self, value): + self.set_zvm_token = value + + def get_zvm_ip(self): + return self.zvm_ip + + def get_zvm_port(self): + return self.zvm_port + + def get_zvm_username(self): + return self.zvm_username + + def get_zvm_password(self): + return self.zvm_password + + def get_zvm_token(self): + return self.zvm_token + + + # vCenter related Get / Set Functions \ No newline at end of file diff --git a/app/zvma9_7/GetStatsFunc.py b/app/zvma9_7/GetStatsFunc.py new file mode 100644 index 0000000..9d694a0 --- /dev/null +++ b/app/zvma9_7/GetStatsFunc.py @@ -0,0 +1,126 @@ + +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +from requests.structures import CaseInsensitiveDict +from tinydb import TinyDB, Query +from tinydbstorage.storage import MemoryStorage +from logging.handlers import RotatingFileHandler + +# Function to get VM Encryption Data from ZVMa version 9.7 +def GetStatsFunc(): + tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging + dbvm = Query() + dbvpg = Query() + while (True) : + global token + global siteId + global siteName + + if (token != ""): + log.info("Got Auth Token!") + log.debug("token: " + str(token)) + log.debug("Stats Collector Loop Running") + + metricsDictionary = {} + + h2 = CaseInsensitiveDict() + h2["Accept"] = "application/json" + h2["Authorization"] = "Bearer " + token + + ## Statistics API + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" + statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + statsapi_json = statsapi.json() + #log.debug(statsapi_json) + + for vm in statsapi_json: + oldvmdata = dict() + + CurrentIops = 0 + CurrentWriteCounterInMBs = 0 + CurrentSyncCounterInMBs = 0 + CurrentNetworkTrafficCounterInMBs = 0 + CurrentEncryptedLBs = 0 + CurrentUnencryptedLBs = 0 + CurrentTotalLBs = 0 + CurrentPercentEncrypted = 0 + VMName = "NA" + + oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + + log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) + if (oldvmdata): + log.info(vm['VmIdentifier'] + " Record Found, Updating DB") + log.debug(oldvmdata[0]) + log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) + + log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") + VMName = oldvmdata[0]['VmName'] + log.debug("Current VM " + str(VMName)) + CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) + log.debug("CurrentIops " + str(CurrentIops)) + CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) + log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) + CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) + log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) + CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) + log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) + CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) + log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) + CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) + log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) + if CurrentTotalLBs != 0: + CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + else: + CurrentPercentEncrypted = 0 + log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + + else: + log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") + #insert original VM record to tempdb + log.debug(tempdb.insert(vm)) + + # update database with VM name, for easier display in Grafana Legends + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] + try: + vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + vapi_json = vapi.json() + except Exception as e: + log.error("Error while sending api request: " + str(e)) + VMName = "Unknown" + else: + log.debug("vapi_json: " + str(vapi_json)) + tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) + log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) + VMName = vapi_json['VmName'] + + # Store Calculated Metrics + metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops + metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs + metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs + metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs + metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs + metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs + metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs + metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + file_object = open('statsmetrics', 'w') + txt_object = open('statsmetrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + file_object.close() + txt_object.close() + + log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") + sleep(scrape_speed) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) \ No newline at end of file diff --git a/app/zvma9_7/__init__.py b/app/zvma9_7/__init__.py new file mode 100644 index 0000000..0231b62 --- /dev/null +++ b/app/zvma9_7/__init__.py @@ -0,0 +1,3 @@ +print("Initializing zvma9_7 package...") + +from .GetStatsFunc import GetStatsFunc \ No newline at end of file From c6ee202b6d8fd23fe4694b25c23bb476763668c7 Mon Sep 17 00:00:00 2001 From: recklessop Date: Tue, 24 Oct 2023 20:42:47 -0400 Subject: [PATCH 02/12] updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 86fa5b1..39d7a74 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ app/threads.txt app/vrametrics app/vrametrics.txt app/__pycache__/* +app/microsoft.gpg app/logs/* app/zvma10/__pycache__/* app/zvma9_7/__pycache__/* \ No newline at end of file From 4781b942f4bb840326abd9fa27dce18366789207 Mon Sep 17 00:00:00 2001 From: recklessop Date: Tue, 24 Oct 2023 20:57:41 -0400 Subject: [PATCH 03/12] started zvm site class from vcenter template --- app/zvma10/zvma.py | 264 +++++++++++++++++++++++++++++++++------------ 1 file changed, 198 insertions(+), 66 deletions(-) diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py index 7e8cb1d..141d788 100644 --- a/app/zvma10/zvma.py +++ b/app/zvma10/zvma.py @@ -1,83 +1,215 @@ # Class for holding variables related to a site. +from pyVim.connect import SmartConnect, Disconnect +from pyVmomi import vim, vmodl +import ssl +import datetime +import logging +from logging.handlers import RotatingFileHandler + +class zvmsite: + def __init__(self, host, username, password, clientid="zerto-client", clientsecret=None, port=443, verify_ssl=False, loglevel="INFO"): + self.host = host + self.port = port + self.username = username + self.password = password + self.verify_ssl = verify_ssl + + self.clientid = None + self.clientsecret = None + self.authtoken = None + self.refreshtoken = None -class site: - def __init__(self): - self.zvm_ip = None - self.zvm_port = 443 - self.zvm_client = None - self.zvm_secret = None - self.zvm_verify_ssl = False - self.zvm_token = None self.id = None self.name = None - self.zvm_version = None + + self.version = None self.zvm_version_major = None self.zvm_version_minor = None self.zvm_version_update = None self.zvm_version_patch = None - self.vc_ip = None - self.vc_port = 443 - self.vc_username = None - self.vc_password = None - self.vc_verify_ssl = False - self.vc_version = None + self.__conn__ = None + self.LOGLEVEL = loglevel.upper() + + #set log line format including container_id + log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") + log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5) + log_handler.setFormatter(log_formatter) + self.log = logging.getLogger("Node-Exporter") + self.log.setLevel(self.LOGLEVEL) + self.log.addHandler(log_handler) + + def connect(self): + self.log.info(f"Log Level set to {self.LOGLEVEL}") + if self.__conn__ is None: + context = ssl.create_default_context() + if not self.verify_ssl: + print("dont verify SSL") + # Create an SSL context without certificate verification + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + # connect to vCenter Server + si = None + try: + self.__conn__ = SmartConnect(host=self.host, user=self.username, pwd=self.password, sslContext=context) + about_info = self.__conn__.content.about + version = about_info.version + self.version = version + self.log.debug("Connected to vCenter Server %s", self.host) + except Exception as e: + self.log.error(f"Error connecting to vCenter Server: {e}") + + def version(self): + return self.version + + def get_cpu_mem_used(self, vra): + if vra == None: + self.log.debug("Get_cpu_mem_used called with no vm name...returning no data") + return + if self.__conn__ == None: + self.log.debug("Trying to get VRA stats without vCenter connection, trying to connect") + self.connect() + + # get the root folder of the vCenter Server + try: + content = self.__conn__.RetrieveContent() + root_folder = content.rootFolder + except: + self.log.debug("Could not get content from vCenter when trying to get VRA stats") + + # create a view for all VMs on the vCenter Server + view_manager = content.viewManager + vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True) + + vm = None + for vm_obj in vm_view.view: + if str(vm_obj.name) == str(vra): + vm = vm_obj + if vm is not None: + self.log.debug(f"Found VRA VM in vCenter with name {vm.name}") + # get the CPU usage and memory usage for the VM + cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage + memory_usage_mb = vm.summary.quickStats.guestMemoryUsage + + # print the CPU and memory usage for the VM + self.log.info(f"VM {vm.name} has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + return [cpu_usage_mhz, memory_usage_mb] + else: + self.log.debug(f"{vm_obj.name} is not a VRA") + raise ValueError("No VRA Found") + + def get_write_iops(self, vm): + try: + content = self.__conn__.RetrieveContent() + except: + self.log.debug("Could not get content from vCenter when trying to get VRA stats") + + # Find the virtual machine by name + vm_name = str(vm) + vm = None + + for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: + if obj.name == vm_name: + vm = obj + break + + if vm is None: + print(f"Virtual machine '{vm_name}' not found") + return + + # Get performance manager + perf_manager = content.perfManager + + # Define the metric ID for write IOPS (counterId = 6) + metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="") + + # calculate the last 60 seconds + end_time = datetime.datetime.now() + start_time = end_time - datetime.timedelta(seconds=60) + + # Create a query specification for roll-up data + query_spec = vim.PerformanceManager.QuerySpec( + entity=vm, + metricId=[metric_id], + format="normal", + startTime=start_time, + endTime=end_time, + intervalId=20, # Use an appropriate interval for the roll-up data + ) - # ZVM Related Set / Get Functions - def set_zvm_ip(self, value): - self.zvm_ip = value + # Query the performance statistics + result = perf_manager.QueryStats(querySpec=[query_spec]) - def set_zvm_port(self, value): - self.zvm_port = value - - def set_zvm_client(self, value): - self.client = value - - def set_zvm_secret(self, value): - self.zvm_secret = value - - def set_zvm_verify_ssl(self, value): - self.zvm_verify_ssl = value - - def set_zvm_token(self, value): - self.set_zvm_token = value - - def set_zvm_id(self, value): - self.id = value - - def set_zvm_name(self, value): - self.zvm_name = value - - def set_zvm_version(self, value): - # Set main zvm version variable - self.zvm_version = value - - # Break out ZVM version string into Major, Minor, Update, Patch variables - self.zvm_version_major, self.zvm_version_minor, temp = self.zvm_version.split(".") - self.zvm_version_update = temp[0] - if (len(temp) > 1): - self.zvm_version_patch = temp[1] + if result: + # Get the average write IOPS for the last 60 seconds + average_write_iops = sum(result[0].value[0].value) / len(result[0].value[0].value) + print(f"Average write IOPS for the last 60 seconds for {vm_name}: {average_write_iops}") + return average_write_iops else: - self.zvm_version_patch = "0" + return None + + def get_average_write_latency(self, vm): + try: + content = self.__conn__.RetrieveContent() + except: + self.log.debug("Could not get content from vCenter when trying to get VM stats") - def set_zvm_token(self, value): - self.set_zvm_token = value + # Find the virtual machine by name + vm_name = str(vm) + vm = None - def get_zvm_ip(self): - return self.zvm_ip + for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: + if obj.name == vm_name: + vm = obj + break - def get_zvm_port(self): - return self.zvm_port - - def get_zvm_username(self): - return self.zvm_username - - def get_zvm_password(self): - return self.zvm_password - - def get_zvm_token(self): - return self.zvm_token - + if vm is None: + self.log.debug(f"Virtual machine '{vm_name}' not found") + return None - # vCenter related Get / Set Functions \ No newline at end of file + # Get performance manager + perf_manager = content.perfManager + + # Define the metric ID for write latency (counterId = X) - replace X with the correct counter ID + # You'll need to find the specific counter ID for write latency in your vSphere environment. + # The counter for write latency may vary based on your configuration. + + metric_id = vim.PerformanceManager.MetricId(counterId=10, instance="") # Replace X with the correct counter ID + + end_time = datetime.datetime.now() + start_time = end_time - datetime.timedelta(seconds=60) + + # Create a query specification for roll-up data + query_spec = vim.PerformanceManager.QuerySpec( + entity=vm, + metricId=[metric_id], + format="normal", + startTime=start_time, + endTime=end_time, + intervalId=20, # Use an appropriate interval for the roll-up data + ) + + # Query the performance statistics + result = perf_manager.QueryStats(querySpec=[query_spec]) + + if result: + # Get the average write latency for the last 60 seconds + if result[0].value[0].value: + average_write_latency = sum(result[0].value[0].value) / len(result[0].value[0].value) + self.log.info(f"Average write latency for the last 60 seconds for {vm_name}: {average_write_latency}") + return average_write_latency + + return None + + + def disconnect(self): + if self.__conn__ == None: + self.log.debug(f"vCenter disconnect requested, but not currently connected.") + return + # Disconnect from vCenter + Disconnect(self.__conn__) + self.__conn__ = None + self.version = None + self.log.debug(f"Disconnected from vCenter") \ No newline at end of file From b51a7bd068a771b23b7d730f2bd162032a339305 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 26 Oct 2023 19:35:29 -0400 Subject: [PATCH 04/12] added zvma class to module --- app/zvma10/__init__.py | 3 +- app/zvma10/zvma.py | 254 +++++++++++++++++++++++++++++++++++------ 2 files changed, 223 insertions(+), 34 deletions(-) diff --git a/app/zvma10/__init__.py b/app/zvma10/__init__.py index a5312ec..bc546f1 100644 --- a/app/zvma10/__init__.py +++ b/app/zvma10/__init__.py @@ -1,4 +1,5 @@ print("Initializing zvma10 package...") #from .zvma import zvm -from .vcenter import vcsite \ No newline at end of file +from .vcenter import vcsite +from .zvma import zvmsite \ No newline at end of file diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py index 141d788..8148e05 100644 --- a/app/zvma10/zvma.py +++ b/app/zvma10/zvma.py @@ -1,34 +1,59 @@ # Class for holding variables related to a site. -from pyVim.connect import SmartConnect, Disconnect -from pyVmomi import vim, vmodl +#from pyVim.connect import SmartConnect, Disconnect +#from pyVmomi import vim, vmodl +import threading +import atexit import ssl import datetime import logging +import requests +from time import sleep +from requests.packages.urllib3.exceptions import InsecureRequestWarning +from requests.structures import CaseInsensitiveDict from logging.handlers import RotatingFileHandler class zvmsite: - def __init__(self, host, username, password, clientid="zerto-client", clientsecret=None, port=443, verify_ssl=False, loglevel="INFO"): + def __init__(self, host, username=None, password=None, port=443, verify_ssl=False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug"): self.host = host self.port = port self.username = username self.password = password self.verify_ssl = verify_ssl + self.uri = "https://" + str(self.host) + ":" + str(self.port) - self.clientid = None - self.clientsecret = None - self.authtoken = None - self.refreshtoken = None + self.client_id = client_id + self.client_secret = None + self.grant_type = grant_type - self.id = None - self.name = None + self.__auththread__ = None + self.token = None + self.expiresIn = 0 + self.token_expire_time = None - self.version = None - self.zvm_version_major = None - self.zvm_version_minor = None - self.zvm_version_update = None - self.zvm_version_patch = None + self.site_id = None + self.site_name = None + self.site_type = None + self.site_type_version = None - self.__conn__ = None + self.zvm_version = dict( + full=None, + major=None, + minor=None, + update=None, + patch=None + ) + + self.api_version = dict( + major = None, + minor = None, + update = None + ) + + self.apiheader = CaseInsensitiveDict() + self.apiheader["Accept"] = "application/json" + + self.__connected__ = False + self._running = False self.LOGLEVEL = loglevel.upper() #set log line format including container_id @@ -39,9 +64,27 @@ class zvmsite: self.log.setLevel(self.LOGLEVEL) self.log.addHandler(log_handler) + atexit.register(self.terminate) + self._running = True + + def terminate(self): + self.log.debug("Terminating other threads") + self._running = False + self.__auththread__.join() + def connect(self): + if (self.__auththread__ == None) or (not self.__auththread__.is_alive()): + self._running = True + self.__auththread__ = threading.Thread(target=self.__authhandler__) + self.__auththread__.start() + self.log.info(f"Starting authentication thread {self.__auththread__.ident}") + else: + self.log.info("Already connected to the ZVM") + + + def __authhandler__(self): self.log.info(f"Log Level set to {self.LOGLEVEL}") - if self.__conn__ is None: + if not self.__connected__: context = ssl.create_default_context() if not self.verify_ssl: print("dont verify SSL") @@ -49,19 +92,139 @@ class zvmsite: context.check_hostname = False context.verify_mode = ssl.CERT_NONE - # connect to vCenter Server - si = None try: - self.__conn__ = SmartConnect(host=self.host, user=self.username, pwd=self.password, sslContext=context) - about_info = self.__conn__.content.about - version = about_info.version - self.version = version - self.log.debug("Connected to vCenter Server %s", self.host) - except Exception as e: - self.log.error(f"Error connecting to vCenter Server: {e}") + # connect to zvm Server + retries = 0 + while self._running: + if self.expiresIn < 30: + self.log.debug(f"Trying login with the following: grant_type: {self.grant_type}, username: {self.username}, password: {self.password}, client_id: {self.client_id}") + h = CaseInsensitiveDict() + h["Content-Type"] = "application/x-www-form-urlencoded" + d = CaseInsensitiveDict() + d["grant_type"] = self.grant_type + if self.grant_type == "password": + d["client_id"] = self.client_id + d["username"] = self.username + d["password"] = self.password + elif self.grant_type == "client_credentials": + d["client_id"] = self.client_id + d["client_secret"] = self.client_secret + else: + self.__connected__ = False + self.log.error(f"Error connection credentials not defined") + + uri = "https://" + str(self.host) + ":" + str(self.port) + "/auth/realms/zerto/protocol/openid-connect/token" + delay = 0 + + try: + response = requests.post(url=uri, data=d, headers=h, verify=self.verify_ssl) + response.raise_for_status() + except requests.exceptions.RequestException as e: + retries += 1 + delay = 2 ** retries + self.log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") + sleep(delay) + continue + else: + retries = 0 + + responseJSON = response.json() + if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: + self.log.error("Authentication response does not contain expected keys") + delay = 2 ** retries + self.__connected__ = False + sleep(delay) + retries += 1 + continue + + self.token = str(responseJSON.get('access_token')) + self.apiheader["Authorization"] = "Bearer " + self.token + self.expiresIn = int(responseJSON.get('expires_in')) + self.log.info("Authentication successful. Token expires in " + str(self.expiresIn) + " seconds") + self.__connected__ = True + + if response.status_code != 200: + self.log.error("Authentication request failed with status code " + str(response.status_code)) + delay = 2 ** retries + self.__connected__ = False + sleep(delay) + retries += 1 + continue + self.log.debug("Connected to ZVM Server %s", self.host) + else: + if not self._running: + self.__auththread__.terminate() + self.log.debug(f"Time till token expiration: {self.expiresIn} seconds") + self.log.debug(f"Current auth token: {self.token}") + sleep(10) + self.expiresIn = self.expiresIn - 10 + + except Exception as e: + self.__connected__ = False + self.log.error(f"Error connecting to ZVM Server: {e}") + + def __set_zvm_version__(self): + # Get Site ID and Name + uri = self.uri + "/v1/localsite" + delay = 0 + try: + self.log.debug("Getting Site ID and Name") + + response = requests.get(url=uri, timeout=3, headers=self.apiheader, verify=self.verify_ssl) + response.raise_for_status() + except requests.exceptions.RequestException as e: + retries += 1 + delay = 2 ** retries + self.log.error("Error while sending api request: " + str(e)) + else: + retries = 0 + + responseJSON = response.json() + self.log.debug(responseJSON) + if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: + self.log.error("LocalSite API response does not contain expected keys") + delay = 2 ** retries + #sleep(delay) + retries += 1 + else: + self.site_id = str(responseJSON.get('SiteIdentifier')) + self.site_name = str(responseJSON.get('SiteName')) + self.zvm_version['full'] = str(responseJSON.get('Version')) + self.site_type_version = str(responseJSON.get('SiteTypeVersion')) + self.site_type = str(responseJSON.get('SiteType')) + + # Break out ZVM version strings + self.zvm_version['major'], self.zvm_version['minor'], temp = self.zvm_version['full'].split(".") + self.zvm_version['update'] = temp[0] + if (len(temp) > 1): + self.zvm_version['patch'] = temp[1] + else: + self.zvm_version['patch'] = "0" + + self.log.info("Site ID: " + self.site_id + " Site Name: " + self.site_name + " Site Type: " + self.site_type ) + def version(self): - return self.version + if self.__connected__ and self._running: + if self.zvm_version['full'] == None: + self.__set_zvm_version__() + return self.zvm_version + else: + return "Error: Not Connected to ZVM" + + ''' + def set_version(self): + # Set main zvm version variable + self.zvm_version = value + + # Break out ZVM version string into Major, Minor, Update, Patch variables + self.zvm_version_major, self.zvm_version_minor, temp = self.zvm_version.split(".") + self.zvm_version_update = temp[0] + if (len(temp) > 1): + self.zvm_version_patch = temp[1] + else: + self.zvm_version_patch = "0" + def get_cpu_mem_used(self, vra): if vra == None: @@ -203,13 +366,38 @@ class zvmsite: return None + ''' def disconnect(self): - if self.__conn__ == None: - self.log.debug(f"vCenter disconnect requested, but not currently connected.") + if self._running == False: + self.log.debug(f"ZVM disconnect requested, but not currently connected.") return - # Disconnect from vCenter - Disconnect(self.__conn__) - self.__conn__ = None - self.version = None - self.log.debug(f"Disconnected from vCenter") \ No newline at end of file + + self.terminate() + # clear class variables + self._running = False + self.__connected__ = False + self.__auththread__ = None + self.token = None + self.expiresIn = 0 + self.token_expire_time = None + + self.site_id = None + self.site_name = None + self.site_type = None + self.site_type_version = None + + self.zvm_version = dict( + major=None, + minor=None, + update=None, + patch=None + ) + + self.api_version = dict( + major = None, + minor = None, + update = None + ) + + self.log.debug(f"Disconnected from ZVM") \ No newline at end of file From 0911a4eccc6c71cee7ea49da67442d174226e244 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Fri, 8 Dec 2023 11:05:50 -0500 Subject: [PATCH 05/12] zvma10 wrapper module Created a python module that has everything needed for comms with ZVMA version 10 in a seperate class. It does not have full api coverage, but is enough to start refactoring the exporter to use the class for ZVMa10 --- .gitignore | 3 +- app/python-node-exporter.py | 3 + app/uuid.txt | 1 + app/version.py | 2 +- app/vmware/__init__.py | 5 + app/{zvma10 => vmware}/vcenter.py | 0 app/vmware/version.py | 10 + app/zvma10/__init__.py | 1 - app/zvma10/test.py | 26 + app/zvma10/version.py | 10 + app/zvma10/zvma.py | 1093 ++++++++++++++++++++--------- 11 files changed, 806 insertions(+), 348 deletions(-) create mode 100644 app/uuid.txt create mode 100644 app/vmware/__init__.py rename app/{zvma10 => vmware}/vcenter.py (100%) create mode 100644 app/vmware/version.py create mode 100644 app/zvma10/test.py create mode 100644 app/zvma10/version.py diff --git a/.gitignore b/.gitignore index 39d7a74..52fc19b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ app/__pycache__/* app/microsoft.gpg app/logs/* app/zvma10/__pycache__/* -app/zvma9_7/__pycache__/* \ No newline at end of file +app/zvma9_7/__pycache__/* +app/temp.sh diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index 383ed5a..897365f 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -16,8 +16,11 @@ from tinydb import TinyDB, Query from tinydbstorage.storage import MemoryStorage from version import VERSION from zvma10.vcenter import vcsite +from zvma10.zvma import zvmsite +from posthog import Posthog requests.packages.urllib3.disable_warnings(InsecureRequestWarning) +callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') zvm_port = os.environ.get('ZVM_PORT', '443') diff --git a/app/uuid.txt b/app/uuid.txt new file mode 100644 index 0000000..6905ced --- /dev/null +++ b/app/uuid.txt @@ -0,0 +1 @@ +c50dfc5a-bbd3-49e4-b8f8-688b19e89960 \ No newline at end of file diff --git a/app/version.py b/app/version.py index 886aefd..3a082fc 100644 --- a/app/version.py +++ b/app/version.py @@ -1,5 +1,5 @@ # version.py -VERSION = "1.3.0" +VERSION = "2.0.0" def main(): # Put your main program code here diff --git a/app/vmware/__init__.py b/app/vmware/__init__.py new file mode 100644 index 0000000..bc546f1 --- /dev/null +++ b/app/vmware/__init__.py @@ -0,0 +1,5 @@ +print("Initializing zvma10 package...") + +#from .zvma import zvm +from .vcenter import vcsite +from .zvma import zvmsite \ No newline at end of file diff --git a/app/zvma10/vcenter.py b/app/vmware/vcenter.py similarity index 100% rename from app/zvma10/vcenter.py rename to app/vmware/vcenter.py diff --git a/app/vmware/version.py b/app/vmware/version.py new file mode 100644 index 0000000..eb85948 --- /dev/null +++ b/app/vmware/version.py @@ -0,0 +1,10 @@ +# version.py +VERSION = "0.1.0" + +def main(): + # Put your main program code here + print(VERSION) + +if __name__ == '__main__': + # This code will be executed only when the file is run as the main program + main() \ No newline at end of file diff --git a/app/zvma10/__init__.py b/app/zvma10/__init__.py index bc546f1..bac090e 100644 --- a/app/zvma10/__init__.py +++ b/app/zvma10/__init__.py @@ -1,5 +1,4 @@ print("Initializing zvma10 package...") #from .zvma import zvm -from .vcenter import vcsite from .zvma import zvmsite \ No newline at end of file diff --git a/app/zvma10/test.py b/app/zvma10/test.py new file mode 100644 index 0000000..e1bf493 --- /dev/null +++ b/app/zvma10/test.py @@ -0,0 +1,26 @@ +import requests +from requests import Request, Session + +# Create a Request object +url = 'http://httpbin.org/post' +headers = {'Content-Type': 'application/json'} +data = {'key': 'value'} +req = Request('POST', url, data=data, headers=headers) + +# Prepare the request +prepared_req = req.prepare() + +# Print the prepared request details +print("Prepared Request:") +print(f"URL: {prepared_req.url}") +print(f"Method: {prepared_req.method}") +print(f"Headers: {prepared_req.headers}") +print(f"Body: {prepared_req.body}") + +# Send the request using a Session +with Session() as s: + response = s.send(prepared_req) + +# Print the response +print(f"\nResponse Status Code: {response.status_code}") +print(response.text) \ No newline at end of file diff --git a/app/zvma10/version.py b/app/zvma10/version.py new file mode 100644 index 0000000..eb85948 --- /dev/null +++ b/app/zvma10/version.py @@ -0,0 +1,10 @@ +# version.py +VERSION = "0.1.0" + +def main(): + # Put your main program code here + print(VERSION) + +if __name__ == '__main__': + # This code will be executed only when the file is run as the main program + main() \ No newline at end of file diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py index 8148e05..5a06c00 100644 --- a/app/zvma10/zvma.py +++ b/app/zvma10/zvma.py @@ -1,31 +1,44 @@ -# Class for holding variables related to a site. -#from pyVim.connect import SmartConnect, Disconnect -#from pyVmomi import vim, vmodl -import threading import atexit +import threading import ssl -import datetime +import json +import os +import time import logging +import socket import requests +import urllib3 +from urllib3.exceptions import InsecureRequestWarning +from urllib.parse import urlencode +from urllib.parse import urlparse from time import sleep -from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.structures import CaseInsensitiveDict from logging.handlers import RotatingFileHandler +from posthog import Posthog +import uuid +from requests import Request, Session +from .version import VERSION class zvmsite: - def __init__(self, host, username=None, password=None, port=443, verify_ssl=False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug"): + def __init__(self, host, username=None, password=None, port: int = 443, verify_ssl: bool = False, client_id="zerto-client", client_secret=None, grant_type="password", loglevel="debug", stats: bool = True): + self.stats = stats self.host = host self.port = port self.username = username self.password = password self.verify_ssl = verify_ssl - self.uri = "https://" + str(self.host) + ":" + str(self.port) + self.base_url = f"https://{self.host}:{self.port}" + + if not self.verify_ssl: + # Disable ssl warnings if verify is set to false. + urllib3.disable_warnings(InsecureRequestWarning) self.client_id = client_id self.client_secret = None self.grant_type = grant_type self.__auththread__ = None + self.__version__ = VERSION self.token = None self.expiresIn = 0 self.token_expire_time = None @@ -35,369 +48,759 @@ class zvmsite: self.site_type = None self.site_type_version = None - self.zvm_version = dict( - full=None, - major=None, - minor=None, - update=None, - patch=None - ) + self.zvm_version = dict(full=None, major=None, minor=None, update=None, patch=None) - self.api_version = dict( - major = None, - minor = None, - update = None - ) + self.__user_agent_string__ = f"zerto_python_sdk_jpaul" self.apiheader = CaseInsensitiveDict() self.apiheader["Accept"] = "application/json" + self.apiheader['User-Agent'] = self.__user_agent_string__ self.__connected__ = False self._running = False self.LOGLEVEL = loglevel.upper() - - #set log line format including container_id - log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") - log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5) - log_handler.setFormatter(log_formatter) - self.log = logging.getLogger("Node-Exporter") - self.log.setLevel(self.LOGLEVEL) - self.log.addHandler(log_handler) - - atexit.register(self.terminate) + + self.setup_logging() + atexit.register(self.disconnect) self._running = True - - def terminate(self): - self.log.debug("Terminating other threads") - self._running = False - self.__auththread__.join() - def connect(self): - if (self.__auththread__ == None) or (not self.__auththread__.is_alive()): - self._running = True - self.__auththread__ = threading.Thread(target=self.__authhandler__) - self.__auththread__.start() - self.log.info(f"Starting authentication thread {self.__auththread__.ident}") - else: - self.log.info("Already connected to the ZVM") - + # Get UUID + self.uuid = self.load_or_generate_uuid() + + # Posthog stats setup + if self.stats: + self.setup_posthog() + self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded') + self.log.debug("Sent PostHog Hook") def __authhandler__(self): self.log.info(f"Log Level set to {self.LOGLEVEL}") if not self.__connected__: context = ssl.create_default_context() if not self.verify_ssl: - print("dont verify SSL") - # Create an SSL context without certificate verification + self.log.debug("Disabling SSL verification") context.check_hostname = False context.verify_mode = ssl.CERT_NONE - try: - # connect to zvm Server - retries = 0 - while self._running: - if self.expiresIn < 30: - self.log.debug(f"Trying login with the following: grant_type: {self.grant_type}, username: {self.username}, password: {self.password}, client_id: {self.client_id}") - h = CaseInsensitiveDict() - h["Content-Type"] = "application/x-www-form-urlencoded" - - d = CaseInsensitiveDict() - d["grant_type"] = self.grant_type - if self.grant_type == "password": - d["client_id"] = self.client_id - d["username"] = self.username - d["password"] = self.password - elif self.grant_type == "client_credentials": - d["client_id"] = self.client_id - d["client_secret"] = self.client_secret - else: - self.__connected__ = False - self.log.error(f"Error connection credentials not defined") - - uri = "https://" + str(self.host) + ":" + str(self.port) + "/auth/realms/zerto/protocol/openid-connect/token" - delay = 0 - - try: - response = requests.post(url=uri, data=d, headers=h, verify=self.verify_ssl) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - self.log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: - self.log.error("Authentication response does not contain expected keys") - delay = 2 ** retries - self.__connected__ = False - sleep(delay) - retries += 1 - continue - - self.token = str(responseJSON.get('access_token')) - self.apiheader["Authorization"] = "Bearer " + self.token - self.expiresIn = int(responseJSON.get('expires_in')) - self.log.info("Authentication successful. Token expires in " + str(self.expiresIn) + " seconds") - self.__connected__ = True - - if response.status_code != 200: - self.log.error("Authentication request failed with status code " + str(response.status_code)) - delay = 2 ** retries - self.__connected__ = False - sleep(delay) - retries += 1 - continue - self.log.debug("Connected to ZVM Server %s", self.host) - else: - if not self._running: - self.__auththread__.terminate() - self.log.debug(f"Time till token expiration: {self.expiresIn} seconds") - self.log.debug(f"Current auth token: {self.token}") - sleep(10) - self.expiresIn = self.expiresIn - 10 - - except Exception as e: - self.__connected__ = False - self.log.error(f"Error connecting to ZVM Server: {e}") - - def __set_zvm_version__(self): - # Get Site ID and Name - uri = self.uri + "/v1/localsite" - delay = 0 - try: - self.log.debug("Getting Site ID and Name") - - response = requests.get(url=uri, timeout=3, headers=self.apiheader, verify=self.verify_ssl) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - self.log.error("Error while sending api request: " + str(e)) - else: retries = 0 - - responseJSON = response.json() - self.log.debug(responseJSON) - if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: - self.log.error("LocalSite API response does not contain expected keys") - delay = 2 ** retries - #sleep(delay) - retries += 1 + while self._running: + if self.expiresIn < 30: + self.log.debug(f"Authenticating to the server: {self.host}") + headers = CaseInsensitiveDict() + headers["Content-Type"] = "application/x-www-form-urlencoded" + + data = { + "grant_type": self.grant_type, + "client_id": self.client_id, + "username": self.username, + "password": self.password + } + if self.grant_type == "client_credentials": + data["client_secret"] = self.client_secret + + uri = self.construct_url(path="auth/realms/zerto/protocol/openid-connect/token") + response = self.make_api_request("POST", uri, data=data, headers=headers) + + if response and 'access_token' in response and 'expires_in' in response: + self.token = str(response['access_token']) + self.apiheader["Authorization"] = "Bearer " + self.token + self.expiresIn = int(response['expires_in']) + self.log.info("Authentication successful") + self.__connected__ = True + else: + self.log.error("Authentication failed") + sleep(2 ** retries) + retries += 1 + else: + sleep(10) + self.expiresIn -= 10 else: - self.site_id = str(responseJSON.get('SiteIdentifier')) - self.site_name = str(responseJSON.get('SiteName')) - self.zvm_version['full'] = str(responseJSON.get('Version')) - self.site_type_version = str(responseJSON.get('SiteTypeVersion')) - self.site_type = str(responseJSON.get('SiteType')) + self.log.info("Authentication thread is already running") + print(f"Auth thread already running") + + def setup_logging(self): + container_id = str(socket.gethostname()) + log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") + log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) + log_handler.setFormatter(log_formatter) + self.log = logging.getLogger("Node-Exporter") + self.log.setLevel(self.LOGLEVEL) + self.log.addHandler(log_handler) + + def __redact__(self, data): + sensitive_keys = ["password", "secret", "token"] # Add any other keys that need redaction + redacted_data = {} + + for key, value in data.items(): + if key in sensitive_keys: + redacted_data[key] = "********" + else: + redacted_data[key] = value + + return redacted_data + + def load_or_generate_uuid(self): + uuid_path = 'uuid.txt' + if os.path.exists(uuid_path): + with open(uuid_path, 'r') as file: + saved_uuid = file.read().strip() + try: + return str(uuid.UUID(saved_uuid)) + except ValueError: + pass # Invalid UUID, generate a new one below + + new_uuid = str(uuid.uuid4()) + with open(uuid_path, 'w') as file: + file.write(new_uuid) + return new_uuid + + def setup_posthog(self): + self.posthog = Posthog(project_api_key='phc_HflqUkx9majhzm8DZva8pTwXFRnOn99onA9xPpK5HaQ', host='https://posthog.jpaul.io') + self.posthog.debug = True + self.posthog.identify(distinct_id=self.uuid) + + def construct_url(self, path="", params=None): + full_url = f"{self.base_url}/{path}" + if params: + query_string = urlencode({k: str(v) for k, v in params.items() if v is not None}) + full_url = f"{full_url}?{query_string}" + return full_url + + def deconstruct_url(self, url): + parsed_url = urlparse(url) + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + path = parsed_url.path + + return base_url, path + + def make_api_request(self, method, url, data=None, json_data=None, headers=None, timeout=3, test=None): + try: + headers = headers or {} + start_time = time.time() # Record the start time + if method == "PUT": + # Create a Request object + headers['Content-Type'] = 'application/json' + data = json.dumps(json_data) + req = Request(method, url, data=data, headers=headers) + + # Prepare the request + prepared_req = req.prepare() + + # Print the prepared request details + self.log.debug("Prepared Request:") + self.log.debug(f"URL: {prepared_req.url}") + self.log.debug(f"Method: {prepared_req.method}") + self.log.debug(f"Headers: {prepared_req.headers}") + self.log.debug(f"Body: {prepared_req.body}") + + # Send the request using a Session + with Session() as s: + response = s.send(prepared_req, verify=self.verify_ssl) + + # Print the response + self.log.debug(f"Response Status Code: {response.status_code}") + self.log.debug(response.text) + elif json_data is not None: + # If json_data is provided, serialize it as JSON and set the appropriate header + serialized_data = json.dumps(json_data) + headers['Content-Type'] = 'application/json' + self.log.debug(f"API Request using JSON Body: {serialized_data}") + response = requests.request(method, url, data=serialized_data, headers=headers, timeout=timeout, verify=self.verify_ssl) + else: + # If json_data is not provided, use data as-is + if data: + self.log.debug(f"API Request using Form/Data Body: {self.__redact__(data)}") + response = requests.request(method, url, data=data, headers=headers, timeout=timeout, verify=self.verify_ssl) + + end_time = time.time() + elapsed_time_ms = (end_time - start_time) * 1000 + response.raise_for_status() + self.log.debug(f'API Request: {method} - {url}') + + # Posthog stats setup + if self.stats: + temp_base, temp_path = self.deconstruct_url(url) + self.posthog.capture( self.uuid, 'API REQUEST', + { + "url": temp_base, + "port": self.port, + "endpoint": temp_path, + "method": method, + "response_time_ms": int(elapsed_time_ms), + "verify_ssl": self.verify_ssl, + "grant_type": self.grant_type, + "status_code": str(response.status_code), + "sdk_version": self.__version__ + }) + self.log.debug("Sent PostHog Hook") + + return response.json() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending API request: {e}") + if e.response: + self.log.error(f"Response content: {e.response.text}") + return None + + def connect(self): + if (self.__auththread__ is None) or (not self.__auththread__.is_alive()): + self._running = True + self.__auththread__ = threading.Thread(target=self.__authhandler__, daemon=True) + self.__auththread__.start() + self.log.info(f"Starting authentication thread {self.__auththread__.ident}") + else: + self.log.info("Already connected to the ZVM") + + def disconnect(self): + self.log.debug("Disconnecting") + self._running = False + if self.__auththread__ and self.__auththread__.is_alive(): + self.__auththread__.join(timeout=5) + + def alert(self, alertidentifier=None): + + if alertidentifier is None: + self.log.error("Alert identifier is required for get_vpg function.") + raise ValueError("Alert identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/alerts/{alertidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def alert_dismiss(self, alertidentifier=None): + + if alertidentifier is None: + self.log.error("Alert identifier is required for get_vpg function.") + raise ValueError("Alert identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/alerts/{alertidentifier}/dismiss", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def alert_undismiss(self, alertidentifier=None): + + if alertidentifier is None: + self.log.error("Alert identifier is required for get_vpg function.") + raise ValueError("Alert identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/alerts/{alertidentifier}/undismiss", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def alert_levels(self): + + params = { + } + + uri = self.construct_url(f"v1/alerts/levels", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def alert_entities(self): + + params = { + } + + uri = self.construct_url(f"v1/alerts/entities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def alert_helpidentifiers(self): + + params = { + } + + uri = self.construct_url(f"v1/alerts/helpidentifiers", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, + entity=None, helpidentifier=None, isdismissed: bool = None): + + params = { + 'startdate': startdate, + 'enddate': enddate, + 'vpgid': vpgid, + 'zorgidentifier': zorgidentifier, + 'level': level, + 'entity': entity, + 'helpidentifier': helpidentifier, + 'isdismissed': isdismissed + } + + uri = self.construct_url("v1/alerts", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def datastore(self, datastoreidentifier=None): + + if datastoreidentifier is None: + self.log.error("Datastore identifier is required for get_datastore function.") + raise ValueError("datastore identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/datastores/{datastoreidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def datastores(self, datadtoreidentifier=None): + + params = { + } + + uri = self.construct_url("v1/datastores", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_enable(self): + + params = { + "encryptionDetectionEnabled": True + } + + uri = self.construct_url("v1/encryptionDetection/state", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def encryptiondetection_disable(self): + + params = { + "encryptionDetectionEnabled": False + } + + uri = self.construct_url("v1/encryptionDetection/state", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def encryptiondetection_status(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/state", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_metrics_vms(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/metrics/vms", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_metrics_volumes(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/metrics/volumes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_metrics_vpgs(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/metrics/vpgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_suspected_vms(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/suspected/vms", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_suspected_volumes(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/suspected/volumes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def encryptiondetection_suspected_vpgs(self): + + params = {} + + uri = self.construct_url("v1/encryptionDetection/suspected/vpgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event(self, eventidentifier=None): + + if eventidentifier is None: + self.log.error("Event identifier is required for get event function.") + raise ValueError("Event identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/events/{eventidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_types(self): + + params = { + } + + uri = self.construct_url(f"v1/events/types", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_entities(self): + + params = { + } + + uri = self.construct_url(f"v1/events/entities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_categories(self): + + params = { + } + + uri = self.construct_url(f"v1/events/categories", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def events(self, startdate=None, enddate=None, vpgid=None, sitename=None, zorgidentifier=None, eventtype=None, + entitytype=None, category=None, username=None, alertidentifier=None): + + params = { + 'startdate': startdate, + 'enddate': enddate, + 'vpgid': vpgid, + 'sitename': sitename, + 'zorgidentifier': zorgidentifier, + 'eventtype': eventtype, + 'entitytype': entitytype, + 'category': category, + 'username': username, + 'alertidentifier': alertidentifier + } + + uri = self.construct_url("v1/events", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def local_site(self): + + params = { + } + + uri = self.construct_url(f"v1/localsite", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def local_site_pairing_statues(self): + + params = { + } + + uri = self.construct_url(f"v1/localsite/pairingstatuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def local_site_send_billing(self): + + params = { + } + + uri = self.construct_url(f"v1/localsite/settings/sendusage", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def local_site_banner(self): + + params = { + } + # uri is spelled incorrectly because it is also spelled incorrectly in zerto + uri = self.construct_url(f"v1/localsite/settings/logingbanner", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def local_site_banner_update(self, enabled: bool = None, loginbanner = None): + + params = { + } + + data = { + "isLoginBannerEnabled": enabled, + "loginBanner": loginbanner + } + # uri is spelled incorrectly because it is also spelled incorrectly in zerto + uri = self.construct_url(f"v1/localsite/settings/logingbanner", params) + return self.make_api_request("PUT", uri, json_data=data, headers=self.apiheader) + + def license(self): + + params = { + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def license_delete(self): + + params = { + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("DELETE", uri, headers=self.apiheader) + + def license_apply(self, license=None): + + if license is None: + self.log.error("A license key is required for apply license function.") + raise ValueError("License key is required.") + + params = { + } + + license = { + "licenseKey": license + } + + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) + + def peer_sites(self): + + params = { + } + + uri = self.construct_url(f"v1/peersites", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def peer_site(self, siteidentifier=None): + if siteidentifier is None: + self.log.error("Site identifier is required for get site function.") + raise ValueError("Site identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def peer_sites_pairing_statues(self): + + params = { + } + + uri = self.construct_url(f"v1/peersites/pairingstatuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def peer_site_add(self, hostname=None, port=None, token=None): + missing_params = [param for param, value in [('hostname', hostname), ('port', port), ('token', token)] if value is None] + + if missing_params: + missing_params_str = ", ".join(missing_params) + error_message = f"Missing required parameter(s): {missing_params_str} for pair site function." + self.log.error(error_message) + raise ValueError(error_message) + + params = {} + + data = { + "hostname": hostname, + "port": port, + "token": token + } + + uri = self.construct_url(f"v1/peersites", params) + return self.make_api_request("POST", uri, json_data=data, headers=self.apiheader) + + def peer_site_delete(self, siteidentifier=None, keepdisks: bool = True): + if siteidentifier is None: + self.log.error("Site identifier is required for delete site function.") + raise ValueError("Site identifier is required.") + + params = {} + + data = { + "iskeeptargetdisks": keepdisks + } + + uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) + return self.make_api_request("DELETE", uri, json=data, headers=self.apiheader) + + def peer_site_pairing_token(self): + params = {} + + uri = self.construct_url(f"v1/peersites/generatetoken", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + + + + def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None): + + params = { + 'startedbeforedate': startedbeforedate, + 'startedafterdate': startedafterdate, + 'completedbeforedate': completedbeforedate, + 'completedafterdate': completedafterdate, + 'type': tasktype, + 'status': status + } + + uri = self.construct_url("v1/tasks", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def task(self, taskidentifier=None): + + if taskidentifier is None: + self.log.error("Task identifier is required for function.") + raise ValueError("Task identifier is required.") + + params = {} + + uri = self.construct_url(f"v1/tasks/{taskidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def task_types(self): + + params = { + } + + uri = self.construct_url(f"v1/tasks/types", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg(self, vpgidentifier=None): + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for get_vpg function.") + raise ValueError("VM identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, + recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, + zorgidentifier=None, priority=None, serviceprofileidentifier=None): + + params = { + 'vpgid': vpgid, + 'vpgname': vpgname, + 'vpgstatus': vpgstatus, + 'vpgsubstatus': vpgsubstatus, + 'protectedsitetype': protectedsitetype, + 'recoverysitetype': recoverysitetype, + 'protectedsiteidentifier': protectedsiteidentifier, + 'recoverysiteidentifier': recoverysiteidentifier, + 'zorgidentifier': zorgidentifier, + 'priority': priority, + 'serviceprofileidentifier': serviceprofileidentifier + } + + uri = self.construct_url("v1/vpgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True): + if vpgidentifier is None: + self.log.error("VPG identifier is required for delete_vpg function.") + raise ValueError("VPG identifier is required.") + + # URL with vpgidentifier in the path + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}") + + # Data to be sent in the request body + data = { + "keepRecoveryVolumes": keeprecoveryvolumes, + "force": force + } + + # Make the POST request + return self.make_api_request("POST", uri, data=data, headers=self.apiheader) + + def vms(self, vmidentifier=None, vmname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, + recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, + zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None): + + params = { + 'vmidentifier': vmidentifier, + 'vmname': vmname, + 'vpgstatus': vpgstatus, + 'vpgsubstatus': vpgsubstatus, + 'protectedsitetype': protectedsitetype, + 'recoverysitetype': recoverysitetype, + 'protectedsiteidentifier': protectedsiteidentifier, + 'recoverysiteidentifier': recoverysiteidentifier, + 'zorgname': zorgname, + 'priority': priority, + 'includebackupvms': includebackupvms, + 'includemountedvms': includemountedvms + } + + uri = self.construct_url("v1/vms", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vm(self, vmidentifier=None, vpgidentifier=None, includebackupvms: bool = None, includemountedvms: bool = None): + + if vmidentifier is None: + self.log.error("VM identifier is required for get_vm function.") + raise ValueError("VM identifier is required.") + + params = { + 'vpgidentifier': vpgidentifier, + 'includebackupvms': includebackupvms, + 'includemountedvms': includemountedvms + } + + uri = self.construct_url(f"v1/vms/{vmidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vm_pointintime(self, vmidentifier=None, vpgidentifier=None, includebackupvms: bool = None, includemountedvms: bool = None): + + if vmidentifier is None: + self.log.error("VM identifier is required for get_vm function.") + raise ValueError("VM identifier is required.") + + params = { + 'vpgidentifier': vpgidentifier, + 'includebackupvms': includebackupvms, + 'includemountedvms': includemountedvms + } + + uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsintime", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None): + if volumetype: + valid_volumetypes = ["scratch", "journal", "recovery", "protected", "appliance"] + + # Convert volumetype to lowercase for case-insensitive comparison + volumetype_lower = volumetype.lower() + + if volumetype_lower not in valid_volumetypes: + raise ValueError(f"Invalid volumetype: {volumetype}. Must be one of {', '.join(valid_volumetypes)}") + + params = { + 'volumetype': volumetype, + 'vpgidentifier': vpgidentifier, + 'datastoreidentifier': datastoreidentifier, + 'protectedvmidentifier': protectedvmidentifier, + 'owningvmidentifier': owningvmidentifier + } + + uri = self.construct_url("v1/volumes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def __set_zvm_version__(self): + uri = self.construct_url("v1/localsite") + response = self.make_api_request("GET", uri, headers=self.apiheader) + if response: + self.site_id = str(response.get('SiteIdentifier', '')) + self.site_name = str(response.get('SiteName', '')) + self.zvm_version['full'] = str(response.get('Version', '')) + self.site_type_version = str(response.get('SiteTypeVersion', '')) + self.site_type = str(response.get('SiteType', '')) # Break out ZVM version strings - self.zvm_version['major'], self.zvm_version['minor'], temp = self.zvm_version['full'].split(".") - self.zvm_version['update'] = temp[0] - if (len(temp) > 1): - self.zvm_version['patch'] = temp[1] - else: - self.zvm_version['patch'] = "0" - - self.log.info("Site ID: " + self.site_id + " Site Name: " + self.site_name + " Site Type: " + self.site_type ) - + version_parts = self.zvm_version['full'].split(".") + if len(version_parts) >= 3: + self.zvm_version['major'], self.zvm_version['minor'], temp = version_parts + self.zvm_version['update'] = temp[0] + self.zvm_version['patch'] = temp[1] if len(temp) > 1 else "0" + self.log.info(f"Site ID: {self.site_id}, Site Name: {self.site_name}, Site Type: {self.site_type}") + def version(self): if self.__connected__ and self._running: - if self.zvm_version['full'] == None: + if self.zvm_version['full'] is None: self.__set_zvm_version__() return self.zvm_version else: return "Error: Not Connected to ZVM" - - ''' - def set_version(self): - # Set main zvm version variable - self.zvm_version = value - - # Break out ZVM version string into Major, Minor, Update, Patch variables - self.zvm_version_major, self.zvm_version_minor, temp = self.zvm_version.split(".") - self.zvm_version_update = temp[0] - if (len(temp) > 1): - self.zvm_version_patch = temp[1] - else: - self.zvm_version_patch = "0" - - - def get_cpu_mem_used(self, vra): - if vra == None: - self.log.debug("Get_cpu_mem_used called with no vm name...returning no data") - return - if self.__conn__ == None: - self.log.debug("Trying to get VRA stats without vCenter connection, trying to connect") - self.connect() - - # get the root folder of the vCenter Server - try: - content = self.__conn__.RetrieveContent() - root_folder = content.rootFolder - except: - self.log.debug("Could not get content from vCenter when trying to get VRA stats") - - # create a view for all VMs on the vCenter Server - view_manager = content.viewManager - vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True) - - vm = None - for vm_obj in vm_view.view: - if str(vm_obj.name) == str(vra): - vm = vm_obj - if vm is not None: - self.log.debug(f"Found VRA VM in vCenter with name {vm.name}") - # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage - memory_usage_mb = vm.summary.quickStats.guestMemoryUsage - - # print the CPU and memory usage for the VM - self.log.info(f"VM {vm.name} has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") - return [cpu_usage_mhz, memory_usage_mb] - else: - self.log.debug(f"{vm_obj.name} is not a VRA") - raise ValueError("No VRA Found") - - def get_write_iops(self, vm): - try: - content = self.__conn__.RetrieveContent() - except: - self.log.debug("Could not get content from vCenter when trying to get VRA stats") - - # Find the virtual machine by name - vm_name = str(vm) - vm = None - - for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: - if obj.name == vm_name: - vm = obj - break - - if vm is None: - print(f"Virtual machine '{vm_name}' not found") - return - - # Get performance manager - perf_manager = content.perfManager - - # Define the metric ID for write IOPS (counterId = 6) - metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="") - - # calculate the last 60 seconds - end_time = datetime.datetime.now() - start_time = end_time - datetime.timedelta(seconds=60) - - # Create a query specification for roll-up data - query_spec = vim.PerformanceManager.QuerySpec( - entity=vm, - metricId=[metric_id], - format="normal", - startTime=start_time, - endTime=end_time, - intervalId=20, # Use an appropriate interval for the roll-up data - ) - - - # Query the performance statistics - result = perf_manager.QueryStats(querySpec=[query_spec]) - - if result: - # Get the average write IOPS for the last 60 seconds - average_write_iops = sum(result[0].value[0].value) / len(result[0].value[0].value) - print(f"Average write IOPS for the last 60 seconds for {vm_name}: {average_write_iops}") - return average_write_iops - else: - return None - - def get_average_write_latency(self, vm): - try: - content = self.__conn__.RetrieveContent() - except: - self.log.debug("Could not get content from vCenter when trying to get VM stats") - - # Find the virtual machine by name - vm_name = str(vm) - vm = None - - for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: - if obj.name == vm_name: - vm = obj - break - - if vm is None: - self.log.debug(f"Virtual machine '{vm_name}' not found") - return None - - # Get performance manager - perf_manager = content.perfManager - - # Define the metric ID for write latency (counterId = X) - replace X with the correct counter ID - # You'll need to find the specific counter ID for write latency in your vSphere environment. - # The counter for write latency may vary based on your configuration. - - metric_id = vim.PerformanceManager.MetricId(counterId=10, instance="") # Replace X with the correct counter ID - - end_time = datetime.datetime.now() - start_time = end_time - datetime.timedelta(seconds=60) - - # Create a query specification for roll-up data - query_spec = vim.PerformanceManager.QuerySpec( - entity=vm, - metricId=[metric_id], - format="normal", - startTime=start_time, - endTime=end_time, - intervalId=20, # Use an appropriate interval for the roll-up data - ) - - # Query the performance statistics - result = perf_manager.QueryStats(querySpec=[query_spec]) - - if result: - # Get the average write latency for the last 60 seconds - if result[0].value[0].value: - average_write_latency = sum(result[0].value[0].value) / len(result[0].value[0].value) - self.log.info(f"Average write latency for the last 60 seconds for {vm_name}: {average_write_latency}") - return average_write_latency - - return None - - ''' - - def disconnect(self): - if self._running == False: - self.log.debug(f"ZVM disconnect requested, but not currently connected.") - return - - self.terminate() - # clear class variables - self._running = False - self.__connected__ = False - self.__auththread__ = None - self.token = None - self.expiresIn = 0 - self.token_expire_time = None - - self.site_id = None - self.site_name = None - self.site_type = None - self.site_type_version = None - - self.zvm_version = dict( - major=None, - minor=None, - update=None, - patch=None - ) - - self.api_version = dict( - major = None, - minor = None, - update = None - ) - - self.log.debug(f"Disconnected from ZVM") \ No newline at end of file From deae17aca9df7bc8c4023b91dd6fff86a83c4375 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Tue, 19 Dec 2023 12:18:46 -0500 Subject: [PATCH 06/12] updates to modules --- app/exporter.py | 719 ++++++++++++++++++ app/python-node-exporter.py | 3 +- app/vmware/__init__.py | 3 +- .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 238 bytes .../__pycache__/vcenter.cpython-310.pyc | Bin 0 -> 5233 bytes app/vmware/vcenter.py | 1 - app/zvma10/zvma.py | 115 +-- 7 files changed, 792 insertions(+), 49 deletions(-) create mode 100644 app/exporter.py create mode 100644 app/vmware/__pycache__/__init__.cpython-310.pyc create mode 100644 app/vmware/__pycache__/vcenter.cpython-310.pyc diff --git a/app/exporter.py b/app/exporter.py new file mode 100644 index 0000000..4bfc477 --- /dev/null +++ b/app/exporter.py @@ -0,0 +1,719 @@ +import requests +import http.server +import socketserver +import os +import ssl +import logging +from logging.handlers import RotatingFileHandler +import threading +import socket +from pyVim.connect import SmartConnect, Disconnect +from pyVmomi import vim +from time import sleep +from requests.packages.urllib3.exceptions import InsecureRequestWarning +from requests.structures import CaseInsensitiveDict +from tinydb import TinyDB, Query +from tinydbstorage.storage import MemoryStorage +from version import VERSION +from vmware.vcenter import vcsite +from zvma10.zvma import zvmsite +from posthog import Posthog +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +""" +Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually +""" + +callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') +verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') +zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') +zvm_port = os.environ.get('ZVM_PORT', '443') +client_id = os.environ.get('CLIENT_ID', 'api-script') +client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') +scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) +api_timeout = int(os.environ.get('API_TIMEOUT', 5)) +LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() +version = str(VERSION) +vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') +vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') +vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') + + +""" +Global Variables used by the program +""" +token = "" +siteId = "NotSet" +siteName = "NotSet" +siteZvmVersion = "" +siteVcVersion = "" +siteZvmMajorVersion = "" +siteZvmMinorVersion = "" +siteZvmUpdateVersion = "" +siteZvmPatchVersion = "" +lastStats = CaseInsensitiveDict() + + +# Authentication Thread which handles authentication and token refresh for ZVM API +def ZvmAuthHandler(): + log.debug("ZVMAuthHandler Thread Started") + expiresIn = 0 + global token + global siteId + global siteName + retries = 0 + while True: + if expiresIn < 30: + h = CaseInsensitiveDict() + h["Content-Type"] = "application/x-www-form-urlencoded" + + d = CaseInsensitiveDict() + d["client_id"] = client_id + d["client_secret"] = client_secret + d["grant_type"] = "client_credentials" + + uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token" + delay = 0 + + try: + response = requests.post(url=uri, data=d, headers=h, verify=verifySSL) + response.raise_for_status() + except requests.exceptions.RequestException as e: + retries += 1 + delay = 2 ** retries + log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") + sleep(delay) + continue + else: + retries = 0 + + responseJSON = response.json() + if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: + log.error("Authentication response does not contain expected keys") + delay = 2 ** retries + sleep(delay) + retries += 1 + continue + + token = str(responseJSON.get('access_token')) + expiresIn = int(responseJSON.get('expires_in')) + log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds") + + if response.status_code != 200: + log.error("Authentication request failed with status code " + str(response.status_code)) + delay = 2 ** retries + sleep(delay) + retries += 1 + continue + + # Get Site ID and Name + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite" + delay = 0 + try: + log.debug("Getting Site ID and Name") + h2 = CaseInsensitiveDict() + h2["Accept"] = "application/json" + h2["Authorization"] = "Bearer " + token + response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + response.raise_for_status() + except requests.exceptions.RequestException as e: + retries += 1 + delay = 2 ** retries + log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") + sleep(delay) + continue + else: + retries = 0 + + responseJSON = response.json() + log.debug(responseJSON) + if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: + log.error("LocalSite API response does not contain expected keys") + delay = 2 ** retries + sleep(delay) + retries += 1 + continue + else: + siteId = str(responseJSON.get('SiteIdentifier')) + siteName = str(responseJSON.get('SiteName')) + siteZvmVersion = str(responseJSON.get('Version')) + siteVcVersion = str(responseJSON.get('SiteTypeVersion')) + + # Break out ZVM version strings + siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") + siteZvmUpdateVersion = siteZvmUpdateVersion[0] + if (len(siteZvmUpdateVersion) > 1): + siteZvmPatchVersion = siteZvmUpdateVersion[1] + else: + siteZvmPatchVersion = "0" + log.info("Site ID: " + siteId + " Site Name: " + siteName) + + expiresIn -= 10 + delay + log.debug("Token Expires in " + str(expiresIn) + " seconds") + sleep(10) + +''' +# Thread which gets VM level encryption statistics from ZVM API + +def GetStatsFunc(): + tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging + dbvm = Query() + dbvpg = Query() + while (True) : + global token + global siteId + global siteName + + if (token != ""): + log.info("Got Auth Token!") + log.debug("token: " + str(token)) + log.debug("Stats Collector Loop Running") + + metricsDictionary = {} + + h2 = CaseInsensitiveDict() + h2["Accept"] = "application/json" + h2["Authorization"] = "Bearer " + token + + ## Statistics API + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" + statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + statsapi_json = statsapi.json() + #log.debug(statsapi_json) + + for vm in statsapi_json: + oldvmdata = dict() + + CurrentIops = 0 + CurrentWriteCounterInMBs = 0 + CurrentSyncCounterInMBs = 0 + CurrentNetworkTrafficCounterInMBs = 0 + CurrentEncryptedLBs = 0 + CurrentUnencryptedLBs = 0 + CurrentTotalLBs = 0 + CurrentPercentEncrypted = 0 + VMName = "NA" + + oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + + log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) + if (oldvmdata): + log.info(vm['VmIdentifier'] + " Record Found, Updating DB") + log.debug(oldvmdata[0]) + log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) + + log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") + VMName = oldvmdata[0]['VmName'] + log.debug("Current VM " + str(VMName)) + CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) + log.debug("CurrentIops " + str(CurrentIops)) + CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) + log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) + CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) + log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) + CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) + log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) + CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) + log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) + CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) + log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) + if CurrentTotalLBs != 0: + CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + else: + CurrentPercentEncrypted = 0 + log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + + else: + log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") + #insert original VM record to tempdb + log.debug(tempdb.insert(vm)) + + # update database with VM name, for easier display in Grafana Legends + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] + try: + vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + vapi_json = vapi.json() + except Exception as e: + log.error("Error while sending api request: " + str(e)) + VMName = "Unknown" + else: + log.debug("vapi_json: " + str(vapi_json)) + tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) + log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) + VMName = vapi_json['VmName'] + + # Store Calculated Metrics + metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops + metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs + metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs + metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs + metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs + metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs + metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs + metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + file_object = open('statsmetrics', 'w') + txt_object = open('statsmetrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + file_object.close() + txt_object.close() + + log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") + sleep(scrape_speed) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) +''' + +# Function which retrieves stats from various ZVM APIs and stores them in a metrics file +def GetDataFunc(): + tempdb = TinyDB(storage=MemoryStorage) + dbvm = Query() + while (True) : + global token + global siteId + global siteName + + if (token != ""): + log.debug("Got Auth Token!") + log.debug("token: " + str(token)) + log.info("Data Collector Loop Running") + + metricsDictionary = {} + + h2 = CaseInsensitiveDict() + h2["Accept"] = "application/json" + h2["Authorization"] = "Bearer " + token + + + ### VPGs API + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/" + service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) + vpg_json = service.json() + #log.debug(vpg_json) + for vpg in vpg_json : + metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] + metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] + metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] + metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] + metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] + metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] + metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] + metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] + metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] + metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] + metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] + metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] + metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] + metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] + + ### Datastores APIs + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/" + service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) + ds_json = service.json() + #log.debug(ds_json) + for ds in ds_json : + + log.debug(f"Processing {ds['DatastoreName']}") + + metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] + metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] + metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] + metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] + metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] + metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] + metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] + + ## VMs API + log.debug("Getting VMs API") + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + + vmapi_json = {} + try: + vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) + vmapi_json = vmapi.json() + except Exception as e: + log.error("Error while sending api request: " + str(e)) + VMName = "Unknown" + return + + log.debug("Got VMs API") + log.debug(vmapi_json) + for vm in vmapi_json : + log.debug("Processing VM: " + str(vm['VmName'])) + log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) + + if siteId == vm['ProtectedSite']['identifier']: + log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") + + if not isinstance(vm["ActualRPO"], int): + vm["ActualRPO"] = -1 + metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] + metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] + metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] + metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] + metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] + metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] + metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] + metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] + metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] + metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] + metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] + log.debug("Processed VM: " + str(vm['VmName'])) + + else: + log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") + + + ## Volumes API for Scratch Volumes + log.debug("Getting Scratch Volumes") + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch" + + volapi_json = {} + try: + volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) + volapi_json = volapi.json() + except Exception as e: + log.error("Error while sending api request: " + str(e)) + VMName = "Unknown" + return + + log.debug("Got Scratch Volumes API") + if(bool(volapi_json)): + for volume in volapi_json : + #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] + # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. + metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" + if (metrickey in metricsDictionary): + metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] + else: + metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] + percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100) + percentage_used = round(percentage_used, 1) + #metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used + + ## Volumes API for Journal Volumes + log.debug("Getting Journal Volumes") + + volapi_json = {} + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal" + try: + volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) + volapi_json = volapi.json() + except Exception as e: + log.error("Error while sending api request: " + str(e)) + VMName = "Unknown" + return + + log.debug("Got Journal Volumes API") + if(bool(volapi_json)): + log.debug("Journal Volumes Exist") + for volume in volapi_json : + log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...") + #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] + # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. + metrickey = "vm_journal_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" + if (metrickey in metricsDictionary): + metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] + else: + metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] + + metrickey = "vm_journal_volume_provisioned_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" + if (metrickey in metricsDictionary): + metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["ProvisionedInBytes"] + else: + metricsDictionary[metrickey] = volume["Size"]["ProvisionedInBytes"] + + metrickey = "vm_journal_volume_count{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" + if (metrickey in metricsDictionary): + metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1 + else: + metricsDictionary[metrickey] = 1 + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + log.debug("Writing metrics to file") + file_object = open('metrics', 'w') + txt_object = open('metrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + + file_object.close() + txt_object.close() + log.debug("Metrics written to file") + + # This function will get data every 10 seconds + log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") + sleep(scrape_speed) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) + +# get VRA CPU and memory usage from vCenter Server +def GetVraMetrics(): + # set up API endpoint and headers + log.debug("GetVraCpuMemory() called") + metricsDictionary = {} + while True: + vra_names = [] + vras = [] + global token + global siteId + global siteName + + log.debug("Checking Token in VRA CPU MEM Collector") + if (token != ""): + log.debug("Auth Token Valid!") + log.debug("token: " + str(token)) + log.info("VRA CPU MEM Collector Running") + + h2 = CaseInsensitiveDict() + h2["Accept"] = "application/json" + h2["Authorization"] = "Bearer " + token + + + ### VRA API + uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras" + + # make API call to get list of VRAs + try: + response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) + except Exception as e: + log.error(f"Error connecting to {uri}: {e}") + return + else: + log.debug("Response from GET /v1/vras: %s", response.text) + # parse JSON response and get the name of each VRA + + vras = response.json() + + log.debug("VRA names: %s", vras) + log.debug(type(vras)) + for vra in vras : + # Gather other VRA Metrics from Zerto API into Metrics Diectionary + metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] + metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] + metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] + metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] + metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] + metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] + metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] + metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] + metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] + + + log.debug("VRA Name: %s", vra['VraName']) + log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") + + # get the CPU and memory usage for each VRA + if is_vcenter_set: + log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") + try: + log.debug("Trying to get stats from vc module") + vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + + # get the CPU usage and memory usage for the VM + cpu_usage_mhz = vradata[0] + memory_usage_mb = vradata[1] + + # print the CPU and memory usage for the VM + log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz + metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb + except: + log.info(f"No VM found with name {vra['VraName']}") + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + file_object = open('vrametrics', 'w') + txt_object = open('vrametrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + + file_object.close() + txt_object.close() + + # This function will get data every 10 seconds + log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") + sleep(scrape_speed * 2) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) + +# function which monitors the threads and restarts them if they die +def ThreadProbe(): + global container_id + while True: + log.debug("Thread Probe Started") + metricsDictionary = {} + + log.debug("Is Auth Thread Alive") + if auth_thread.is_alive(): + metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + else: + metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + + log.debug("Is Data Thread Alive") + if data_thread.is_alive(): + metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + else: + metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + + #log.debug("Is Stats Thread Alive") + #if stats_thread.is_alive(): + # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + #else: + # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + + log.debug("Is VRA Metrics Thread Alive") + if vra_metrics_thread.is_alive(): + metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + else: + metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + + log.debug("Writing Probe data to files") + file_object = open('threads', 'w') + txt_object = open('threads.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + + log.debug("Trying to Close probe txt files") + file_object.close() + txt_object.close() + + log.debug("Probe Thread Going to Sleep") + sleep(30) + +#----------------run http server on port 9999----------------- +def WebServer(): + log.info("Web Server Started") + PORT = 9999 + + Handler = http.server.SimpleHTTPRequestHandler + + with socketserver.TCPServer(("", PORT), Handler) as httpd: + log.info(f"Webserver running on port {PORT}") + httpd.serve_forever() + +def start_thread(target_func): + # start a new thread + thread = threading.Thread(target=target_func) + thread.start() + # return the thread object + return thread + +""" +Main Program Logic +""" + +# Initialize zvmsite instance +zvm_instance = zvmsite( + host=zvm_url, + port=zvm_port, + client_id=client_id, + client_secret=client_secret, + grant_type="client_credentials", + loglevel=LOGLEVEL +) + +# Start the zvmsite authentication thread +zvm_instance.connect() + +# Get the hostname of the machine +container_id = str(socket.gethostname()) + +#set log line format including container_id +log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") +log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) +log_handler.setFormatter(log_formatter) +log = logging.getLogger("Node-Exporter") +log.setLevel(LOGLEVEL) +log.addHandler(log_handler) +log.info(f"Zerto-Node-Exporter - Version {version}") +log.info(f"Log Level: {LOGLEVEL}") +log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) + +# Check if vCenter is set, if not disable VRA metrics +is_vcenter_set = True +if vcenter_host == "vcenter.local": + log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") + is_vcenter_set = False +log.debug("vCenter data collection is enabled") +vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") + + +# start the threads +auth_thread = start_thread(ZvmAuthHandler) +data_thread = start_thread(GetDataFunc) +#stats_thread = start_thread(GetStatsFunc()) +vra_metrics_thread = start_thread(GetVraMetrics) +webserver_thread = start_thread(WebServer) +probe_thread = start_thread(ThreadProbe) + +# loop indefinitely +while True: + # check if any thread has crashed + sleep(10) + if not probe_thread.is_alive(): + # restart the thread + log.error("Probe Thread Died - Restarting") + probe_thread = start_thread(ThreadProbe) + if not auth_thread.is_alive(): + # restart the thread + log.error("Authentication Thread Died - Restarting") + auth_thread = start_thread(ZvmAuthHandler) + if not data_thread.is_alive(): + # restart the thread + log.error("Data Thread Died - Restarting") + data_thread = start_thread(GetDataFunc) + #if not stats_thread.is_alive(): + # # restart the thread + # log.error("Stats Thread Died - Restarting") + # stats_thread = start_thread(GetStatsFunc()) + if not vra_metrics_thread.is_alive(): + # restart the thread + log.error("VRA Metrics Thread Died - Restarting") + vra_metrics_thread = start_thread(GetVraMetrics) + if not webserver_thread.is_alive(): + # restart the thread + log.error("Webserver Thread Died - Restarting") + webserver_thread = start_thread(WebServer) + sleep(api_timeout) \ No newline at end of file diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index 897365f..a6bc628 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -15,7 +15,7 @@ from requests.structures import CaseInsensitiveDict from tinydb import TinyDB, Query from tinydbstorage.storage import MemoryStorage from version import VERSION -from zvma10.vcenter import vcsite +from vmware.vcenter import vcsite from zvma10.zvma import zvmsite from posthog import Posthog @@ -587,7 +587,6 @@ def GetVraMetrics(): log.debug("Waiting 1 second for Auth Token") sleep(1) - # function which monitors the threads and restarts them if they die def ThreadProbe(): global container_id diff --git a/app/vmware/__init__.py b/app/vmware/__init__.py index bc546f1..a5312ec 100644 --- a/app/vmware/__init__.py +++ b/app/vmware/__init__.py @@ -1,5 +1,4 @@ print("Initializing zvma10 package...") #from .zvma import zvm -from .vcenter import vcsite -from .zvma import zvmsite \ No newline at end of file +from .vcenter import vcsite \ No newline at end of file diff --git a/app/vmware/__pycache__/__init__.cpython-310.pyc b/app/vmware/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab8f405b2c8e63e188ccd6d8c8f3e82ee59323ea GIT binary patch literal 238 zcmd1j<>g`kf{K!g)Mz067{oyaOhAqU5En}TiByIZhGs@ah7`sWrX0pxrYI&Ln>m<4 zlch?|GcU6wGchN#Dl;!#p{gu5(a=DlATc>RF+EjJPwyooP`f7MEw-}c;>?m%KTYOa ztOZ4xc_p{l%aT*`N>Yo8KsK#pC}IUtVB(jRerR!OQL%njX>mzrp1yl#iAQOYepG5v zNq)R*ML~WMP?vsUL4kf*Zh2x+s(yTYCeRJ>@p=W7w>WHa^HWN5Qtd$Y7P9~e4h8{c E0LQ64jsO4v literal 0 HcmV?d00001 diff --git a/app/vmware/__pycache__/vcenter.cpython-310.pyc b/app/vmware/__pycache__/vcenter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10f076207ab0100f1e9c9148b15f725749a3bd09 GIT binary patch literal 5233 zcmcgwOLN=S6~>DI2vQXF@N1k&QO9b`X)LvA`iPS{b`;5WN0J>&RvJ!P6yk*>&;&tW zfRcp)JDoba*89w4GD9u>8@lPTzu|7X@~&$y+J5JP)GJOG?M%tRci;D(dvU&V&P6;i z;VAgkZzaJO-&2&oQ{m)KN8uKpWD~#?rgoG_ZK{#h)Kn^K9X&FdMpSB+WLfW+O%wHc z$7)(KwVO6lqf?HYrlTr%6joy9mkKkVY0V1L*Oi9#5_2mJO;pz-pQnp)w;Qxl;d~M% zts;*$eJ_keX)lUdXRV=%nTK)er(t(zIqU>?{VwYSyhYQT{2d9ujM_kHs!VBWY=#-E z^rhO=naQj#m8QXLRz_N44yzzFS(Qy7wOEZ!BDGnaO(89_X$U(j%)M3;roqdI zjC*U#_gmJ9Wy{Ey@GRm<{s_p`T=A6L0IInL)N&oD=Ov)Q6ktiBnVUoFkji#i9y&~A z+MzbEQXCV>%*^Ndz|QSqHMb5`RY9K#h8+6Wa;04xP6BOek)?Vr^Gt!%_FDGd+%-RG zrC}5_k`Lytb%MR1)AdKCbe9J{TN@XnAW8fkbj+saKEFN}U7us_+}#^v&-`f zop>i%Sc!M8uliy4`d%yOrU9QvZ3t~KU()Uyy0EwLZfFT%NB+}~57HnJl`X&ZxZhih z``t8~Sc_S3ed%d0<{00Y62@+vq{5)Gu=`2Ci_twlN%mvTgtHg$aQnbZl8&&DPr|q> zY|kS#@jOxL_j&;r_R9S`D@&V8E5d?EJ5Ul)UXFR>r&xuX5KdH+AU&!Q<((kKZgwzM z#|UAgb%oAHI6h+|?P`>Tkp!J>Uc&^uPB29t_2l1Ct9nfZsx`H)W|M`!&W}g6ptnaqIt`jeD(l~nf$Atjl|iZ6+SkfA+DM&r zrt}rQl^Vm+p)#OWGx>gMG3^=Djp?}xl{MOBqz+RCT6QI^pidRg1fE*1GUGWB0E*w% z+mr43K+pB|)WDFmm_n^GoMs?IyLt%A@)>N)x?(8l%z>{e1K5_VF8jGAm%sFwyF3%31!Cna5`l?#toJ4#|k};?lznZ|(lt zlBkR*0mB$@jix@WpTd$8Vv>@-757s$~ff_MynEXv@F9tnL zg_eX(M-GI}L3lY@4Wtei@Yc9L8EsO=eT4 z{Ui0ih}EE|%43a_;z}Bj`jc{@tjU(kqm~gdhWa6d%p7VT8lZ-jYwglN|4iw^+0lR0 zU#FQqSI}yb(hsd%PwjSj2gEqk2TN6GctmPKk;aK}B zAB<)!v|@t@pIr&;Bh`bV`;6#(|u6~?NxM3oN^J#&sx)#fY19(aJMi_vh3A9E8 z*A8__>EfjPcYOe;QD z?kjf&`WTJ!sJ1MFr_)|7su$yWAwtfmz?Epjd*t4{AaQY`~R`{7HkFCg(D^( z1rS=hDwzy7eS*o8?a5(%#N^5L)F~!Uw5Pv=$up;z{MCrHvy#WN{Hr5ok7~f|^QZef zQ;FG9)(fZEP4b>#cXso?bhxX9d*9&q@W%PC052)f;wuEJ1P=&4C0HleAb3RZ1%c%1 zyQjI@0WG?%ga1fZ{#%UxANlH$0KX&f3APAY1dJdc*e2K|*df5ZSCP*0Tg>BaYI;o2 zA&3aN1TjI6;0d64?lhrBdp*mkOT?+y_WTnjNj?UA7rlr)pqJ@ruc4P?Xm#xxf^DLhB-ozQ7Q~MUB`+* zj0Bfg2L21mT{(ickQ2`F>VHK!p}Q4btPyx&2 zvc331;dmY{YW)u7tDg4+FI*<_9c bool: + # Assuming self.token is the authentication token and it's set upon successful authentication + # and self.__connected__ is a boolean indicating the connection status + return self.token is not None and self.__connected__ + + def setup_logging(self) -> None: container_id = str(socket.gethostname()) log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) @@ -127,7 +135,7 @@ class zvmsite: self.log.setLevel(self.LOGLEVEL) self.log.addHandler(log_handler) - def __redact__(self, data): + def __redact__(self, data) -> str: sensitive_keys = ["password", "secret", "token"] # Add any other keys that need redaction redacted_data = {} @@ -139,7 +147,7 @@ class zvmsite: return redacted_data - def load_or_generate_uuid(self): + def load_or_generate_uuid(self) -> uuid.uuid4: uuid_path = 'uuid.txt' if os.path.exists(uuid_path): with open(uuid_path, 'r') as file: @@ -154,26 +162,26 @@ class zvmsite: file.write(new_uuid) return new_uuid - def setup_posthog(self): + def setup_posthog(self) -> None: self.posthog = Posthog(project_api_key='phc_HflqUkx9majhzm8DZva8pTwXFRnOn99onA9xPpK5HaQ', host='https://posthog.jpaul.io') self.posthog.debug = True self.posthog.identify(distinct_id=self.uuid) - def construct_url(self, path="", params=None): + def construct_url(self, path="", params=None) -> str: full_url = f"{self.base_url}/{path}" if params: query_string = urlencode({k: str(v) for k, v in params.items() if v is not None}) full_url = f"{full_url}?{query_string}" return full_url - def deconstruct_url(self, url): + def deconstruct_url(self, url) -> Tuple[str, str]: parsed_url = urlparse(url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" path = parsed_url.path return base_url, path - def make_api_request(self, method, url, data=None, json_data=None, headers=None, timeout=3, test=None): + def make_api_request(self, method, url, data=None, json_data=None, headers=None, timeout=3, test=None) -> Optional[Union[Dict[str, Any], str]]: try: headers = headers or {} start_time = time.time() # Record the start time @@ -241,7 +249,7 @@ class zvmsite: self.log.error(f"Response content: {e.response.text}") return None - def connect(self): + def connect(self) -> None: if (self.__auththread__ is None) or (not self.__auththread__.is_alive()): self._running = True self.__auththread__ = threading.Thread(target=self.__authhandler__, daemon=True) @@ -250,13 +258,13 @@ class zvmsite: else: self.log.info("Already connected to the ZVM") - def disconnect(self): + def disconnect(self) -> None: self.log.debug("Disconnecting") self._running = False if self.__auththread__ and self.__auththread__.is_alive(): self.__auththread__.join(timeout=5) - def alert(self, alertidentifier=None): + def alert(self, alertidentifier=None) -> Dict[str, Any]: if alertidentifier is None: self.log.error("Alert identifier is required for get_vpg function.") @@ -268,31 +276,53 @@ class zvmsite: uri = self.construct_url(f"v1/alerts/{alertidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def alert_dismiss(self, alertidentifier=None): - + def alert_dismiss(self, alertidentifier=None) -> bool: if alertidentifier is None: - self.log.error("Alert identifier is required for get_vpg function.") + self.log.error("Alert identifier is required for alert_dismiss function.") raise ValueError("Alert identifier is required.") - params = { - } - + params = {} uri = self.construct_url(f"v1/alerts/{alertidentifier}/dismiss", params) - return self.make_api_request("POST", uri, headers=self.apiheader) - - def alert_undismiss(self, alertidentifier=None): - + + try: + response = self.make_api_request("POST", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to dismiss alert: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending dismiss alert request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def alert_undismiss(self, alertidentifier=None) -> bool: if alertidentifier is None: - self.log.error("Alert identifier is required for get_vpg function.") + self.log.error("Alert identifier is required for alert_undismiss function.") raise ValueError("Alert identifier is required.") - params = { - } - + params = {} uri = self.construct_url(f"v1/alerts/{alertidentifier}/undismiss", params) - return self.make_api_request("POST", uri, headers=self.apiheader) - - def alert_levels(self): + + try: + response = self.make_api_request("POST", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to undismiss alert: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending undismiss alert request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def alert_levels(self) -> List[str]: params = { } @@ -300,7 +330,7 @@ class zvmsite: uri = self.construct_url(f"v1/alerts/levels", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def alert_entities(self): + def alert_entities(self) -> List[str]: params = { } @@ -308,7 +338,7 @@ class zvmsite: uri = self.construct_url(f"v1/alerts/entities", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def alert_helpidentifiers(self): + def alert_helpidentifiers(self) -> List[str]: params = { } @@ -317,7 +347,7 @@ class zvmsite: return self.make_api_request("GET", uri, headers=self.apiheader) def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, - entity=None, helpidentifier=None, isdismissed: bool = None): + entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]: params = { 'startdate': startdate, @@ -345,7 +375,7 @@ class zvmsite: uri = self.construct_url(f"v1/datastores/{datastoreidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def datastores(self, datadtoreidentifier=None): + def datastores(self, datadtoreidentifier=None) -> List[Dict[str, Any]]: params = { } @@ -457,7 +487,7 @@ class zvmsite: return self.make_api_request("GET", uri, headers=self.apiheader) def events(self, startdate=None, enddate=None, vpgid=None, sitename=None, zorgidentifier=None, eventtype=None, - entitytype=None, category=None, username=None, alertidentifier=None): + entitytype=None, category=None, username=None, alertidentifier=None) -> List[Dict[str, Any]]: params = { 'startdate': startdate, @@ -553,7 +583,7 @@ class zvmsite: uri = self.construct_url(f"v1/license", params) return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) - def peer_sites(self): + def peer_sites(self) -> List[Dict[str, Any]]: params = { } @@ -620,10 +650,7 @@ class zvmsite: uri = self.construct_url(f"v1/peersites/generatetoken", params) return self.make_api_request("POST", uri, headers=self.apiheader) - - - - def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None): + def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None) -> List[Dict[str, Any]]: params = { 'startedbeforedate': startedbeforedate, @@ -670,7 +697,7 @@ class zvmsite: def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, - zorgidentifier=None, priority=None, serviceprofileidentifier=None): + zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]: params = { 'vpgid': vpgid, @@ -708,7 +735,7 @@ class zvmsite: def vms(self, vmidentifier=None, vmname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, - zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None): + zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None) -> List[Dict[str, Any]]: params = { 'vmidentifier': vmidentifier, @@ -758,7 +785,7 @@ class zvmsite: uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsintime", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None): + def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None) -> List[Dict[str, Any]]: if volumetype: valid_volumetypes = ["scratch", "journal", "recovery", "protected", "appliance"] From 7618fa7d04e06de864a4c334381642434f8d2920 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 21 Dec 2023 10:34:41 -0500 Subject: [PATCH 07/12] finished exporter refactor Ready for some beta testing with Zerto 10.0 --- .gitignore | 1 + app/exporter.py | 719 ---------------- app/python-node-exporter.py | 792 ++++++++---------- app/vmware/__init__.py | 2 +- .../__pycache__/__init__.cpython-310.pyc | Bin 238 -> 239 bytes .../__pycache__/vcenter.cpython-310.pyc | Bin 5233 -> 5337 bytes app/vmware/vcenter.py | 24 +- app/zvma10/zvma.py | 665 +++++++++++---- 8 files changed, 877 insertions(+), 1326 deletions(-) delete mode 100644 app/exporter.py diff --git a/.gitignore b/.gitignore index 52fc19b..5b51a0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ logs/* .env/* .DS_Store +uuid.txt app/metrics app/metrics.txt app/statsmetrics diff --git a/app/exporter.py b/app/exporter.py deleted file mode 100644 index 4bfc477..0000000 --- a/app/exporter.py +++ /dev/null @@ -1,719 +0,0 @@ -import requests -import http.server -import socketserver -import os -import ssl -import logging -from logging.handlers import RotatingFileHandler -import threading -import socket -from pyVim.connect import SmartConnect, Disconnect -from pyVmomi import vim -from time import sleep -from requests.packages.urllib3.exceptions import InsecureRequestWarning -from requests.structures import CaseInsensitiveDict -from tinydb import TinyDB, Query -from tinydbstorage.storage import MemoryStorage -from version import VERSION -from vmware.vcenter import vcsite -from zvma10.zvma import zvmsite -from posthog import Posthog -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -""" -Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually -""" - -callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') -verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') -zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') -zvm_port = os.environ.get('ZVM_PORT', '443') -client_id = os.environ.get('CLIENT_ID', 'api-script') -client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') -scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) -api_timeout = int(os.environ.get('API_TIMEOUT', 5)) -LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() -version = str(VERSION) -vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') -vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') -vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') - - -""" -Global Variables used by the program -""" -token = "" -siteId = "NotSet" -siteName = "NotSet" -siteZvmVersion = "" -siteVcVersion = "" -siteZvmMajorVersion = "" -siteZvmMinorVersion = "" -siteZvmUpdateVersion = "" -siteZvmPatchVersion = "" -lastStats = CaseInsensitiveDict() - - -# Authentication Thread which handles authentication and token refresh for ZVM API -def ZvmAuthHandler(): - log.debug("ZVMAuthHandler Thread Started") - expiresIn = 0 - global token - global siteId - global siteName - retries = 0 - while True: - if expiresIn < 30: - h = CaseInsensitiveDict() - h["Content-Type"] = "application/x-www-form-urlencoded" - - d = CaseInsensitiveDict() - d["client_id"] = client_id - d["client_secret"] = client_secret - d["grant_type"] = "client_credentials" - - uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token" - delay = 0 - - try: - response = requests.post(url=uri, data=d, headers=h, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: - log.error("Authentication response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - token = str(responseJSON.get('access_token')) - expiresIn = int(responseJSON.get('expires_in')) - log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds") - - if response.status_code != 200: - log.error("Authentication request failed with status code " + str(response.status_code)) - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - # Get Site ID and Name - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite" - delay = 0 - try: - log.debug("Getting Site ID and Name") - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - log.debug(responseJSON) - if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: - log.error("LocalSite API response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - else: - siteId = str(responseJSON.get('SiteIdentifier')) - siteName = str(responseJSON.get('SiteName')) - siteZvmVersion = str(responseJSON.get('Version')) - siteVcVersion = str(responseJSON.get('SiteTypeVersion')) - - # Break out ZVM version strings - siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") - siteZvmUpdateVersion = siteZvmUpdateVersion[0] - if (len(siteZvmUpdateVersion) > 1): - siteZvmPatchVersion = siteZvmUpdateVersion[1] - else: - siteZvmPatchVersion = "0" - log.info("Site ID: " + siteId + " Site Name: " + siteName) - - expiresIn -= 10 + delay - log.debug("Token Expires in " + str(expiresIn) + " seconds") - sleep(10) - -''' -# Thread which gets VM level encryption statistics from ZVM API - -def GetStatsFunc(): - tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging - dbvm = Query() - dbvpg = Query() - while (True) : - global token - global siteId - global siteName - - if (token != ""): - log.info("Got Auth Token!") - log.debug("token: " + str(token)) - log.debug("Stats Collector Loop Running") - - metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - ## Statistics API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" - statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - statsapi_json = statsapi.json() - #log.debug(statsapi_json) - - for vm in statsapi_json: - oldvmdata = dict() - - CurrentIops = 0 - CurrentWriteCounterInMBs = 0 - CurrentSyncCounterInMBs = 0 - CurrentNetworkTrafficCounterInMBs = 0 - CurrentEncryptedLBs = 0 - CurrentUnencryptedLBs = 0 - CurrentTotalLBs = 0 - CurrentPercentEncrypted = 0 - VMName = "NA" - - oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) - - log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) - if (oldvmdata): - log.info(vm['VmIdentifier'] + " Record Found, Updating DB") - log.debug(oldvmdata[0]) - log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) - - log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") - VMName = oldvmdata[0]['VmName'] - log.debug("Current VM " + str(VMName)) - CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) - log.debug("CurrentIops " + str(CurrentIops)) - CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) - log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) - CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) - log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) - CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) - log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) - CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) - log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) - CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) - log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) - if CurrentTotalLBs != 0: - CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) - else: - CurrentPercentEncrypted = 0 - log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) - - else: - log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") - #insert original VM record to tempdb - log.debug(tempdb.insert(vm)) - - # update database with VM name, for easier display in Grafana Legends - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] - try: - vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vapi_json = vapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - else: - log.debug("vapi_json: " + str(vapi_json)) - tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) - log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) - VMName = vapi_json['VmName'] - - # Store Calculated Metrics - metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops - metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs - metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs - metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs - metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs - metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs - metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs - metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('statsmetrics', 'w') - txt_object = open('statsmetrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - file_object.close() - txt_object.close() - - log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") - sleep(scrape_speed) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) -''' - -# Function which retrieves stats from various ZVM APIs and stores them in a metrics file -def GetDataFunc(): - tempdb = TinyDB(storage=MemoryStorage) - dbvm = Query() - while (True) : - global token - global siteId - global siteName - - if (token != ""): - log.debug("Got Auth Token!") - log.debug("token: " + str(token)) - log.info("Data Collector Loop Running") - - metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - - ### VPGs API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - vpg_json = service.json() - #log.debug(vpg_json) - for vpg in vpg_json : - metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] - metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] - metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] - metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] - metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] - metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] - metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] - metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] - metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] - metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] - metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] - metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] - metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] - metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] - - ### Datastores APIs - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - ds_json = service.json() - #log.debug(ds_json) - for ds in ds_json : - - log.debug(f"Processing {ds['DatastoreName']}") - - metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] - metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] - metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] - metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] - metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] - metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] - metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] - - ## VMs API - log.debug("Getting VMs API") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" - - vmapi_json = {} - try: - vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vmapi_json = vmapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got VMs API") - log.debug(vmapi_json) - for vm in vmapi_json : - log.debug("Processing VM: " + str(vm['VmName'])) - log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) - - if siteId == vm['ProtectedSite']['identifier']: - log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - - if not isinstance(vm["ActualRPO"], int): - vm["ActualRPO"] = -1 - metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] - metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] - metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] - metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] - metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] - metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] - metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] - metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] - metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] - metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] - metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] - log.debug("Processed VM: " + str(vm['VmName'])) - - else: - log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") - - - ## Volumes API for Scratch Volumes - log.debug("Getting Scratch Volumes") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch" - - volapi_json = {} - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Scratch Volumes API") - if(bool(volapi_json)): - for volume in volapi_json : - #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] - # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. - metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] - percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100) - percentage_used = round(percentage_used, 1) - #metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used - - ## Volumes API for Journal Volumes - log.debug("Getting Journal Volumes") - - volapi_json = {} - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal" - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Journal Volumes API") - if(bool(volapi_json)): - log.debug("Journal Volumes Exist") - for volume in volapi_json : - log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...") - #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] - # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. - metrickey = "vm_journal_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["UsedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["UsedInBytes"] - - metrickey = "vm_journal_volume_provisioned_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + volume["Size"]["ProvisionedInBytes"] - else: - metricsDictionary[metrickey] = volume["Size"]["ProvisionedInBytes"] - - metrickey = "vm_journal_volume_count{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" - if (metrickey in metricsDictionary): - metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1 - else: - metricsDictionary[metrickey] = 1 - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - log.debug("Writing metrics to file") - file_object = open('metrics', 'w') - txt_object = open('metrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() - log.debug("Metrics written to file") - - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(scrape_speed) + " seconds") - sleep(scrape_speed) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) - -# get VRA CPU and memory usage from vCenter Server -def GetVraMetrics(): - # set up API endpoint and headers - log.debug("GetVraCpuMemory() called") - metricsDictionary = {} - while True: - vra_names = [] - vras = [] - global token - global siteId - global siteName - - log.debug("Checking Token in VRA CPU MEM Collector") - if (token != ""): - log.debug("Auth Token Valid!") - log.debug("token: " + str(token)) - log.info("VRA CPU MEM Collector Running") - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - - ### VRA API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras" - - # make API call to get list of VRAs - try: - response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - except Exception as e: - log.error(f"Error connecting to {uri}: {e}") - return - else: - log.debug("Response from GET /v1/vras: %s", response.text) - # parse JSON response and get the name of each VRA - - vras = response.json() - - log.debug("VRA names: %s", vras) - log.debug(type(vras)) - for vra in vras : - # Gather other VRA Metrics from Zerto API into Metrics Diectionary - metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] - metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] - metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] - metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] - metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] - metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] - metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] - metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] - metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - - - log.debug("VRA Name: %s", vra['VraName']) - log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") - - # get the CPU and memory usage for each VRA - if is_vcenter_set: - log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") - try: - log.debug("Trying to get stats from vc module") - vradata = vc_connection.get_cpu_mem_used(vra['VraName']) - - # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vradata[0] - memory_usage_mb = vradata[1] - - # print the CPU and memory usage for the VM - log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") - metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz - metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb - except: - log.info(f"No VM found with name {vra['VraName']}") - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('vrametrics', 'w') - txt_object = open('vrametrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() - - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") - sleep(scrape_speed * 2) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) - -# function which monitors the threads and restarts them if they die -def ThreadProbe(): - global container_id - while True: - log.debug("Thread Probe Started") - metricsDictionary = {} - - log.debug("Is Auth Thread Alive") - if auth_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is Data Thread Alive") - if data_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - #log.debug("Is Stats Thread Alive") - #if stats_thread.is_alive(): - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - #else: - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is VRA Metrics Thread Alive") - if vra_metrics_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Writing Probe data to files") - file_object = open('threads', 'w') - txt_object = open('threads.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - log.debug("Trying to Close probe txt files") - file_object.close() - txt_object.close() - - log.debug("Probe Thread Going to Sleep") - sleep(30) - -#----------------run http server on port 9999----------------- -def WebServer(): - log.info("Web Server Started") - PORT = 9999 - - Handler = http.server.SimpleHTTPRequestHandler - - with socketserver.TCPServer(("", PORT), Handler) as httpd: - log.info(f"Webserver running on port {PORT}") - httpd.serve_forever() - -def start_thread(target_func): - # start a new thread - thread = threading.Thread(target=target_func) - thread.start() - # return the thread object - return thread - -""" -Main Program Logic -""" - -# Initialize zvmsite instance -zvm_instance = zvmsite( - host=zvm_url, - port=zvm_port, - client_id=client_id, - client_secret=client_secret, - grant_type="client_credentials", - loglevel=LOGLEVEL -) - -# Start the zvmsite authentication thread -zvm_instance.connect() - -# Get the hostname of the machine -container_id = str(socket.gethostname()) - -#set log line format including container_id -log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") -log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) -log_handler.setFormatter(log_formatter) -log = logging.getLogger("Node-Exporter") -log.setLevel(LOGLEVEL) -log.addHandler(log_handler) -log.info(f"Zerto-Node-Exporter - Version {version}") -log.info(f"Log Level: {LOGLEVEL}") -log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) - -# Check if vCenter is set, if not disable VRA metrics -is_vcenter_set = True -if vcenter_host == "vcenter.local": - log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") - is_vcenter_set = False -log.debug("vCenter data collection is enabled") -vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") - - -# start the threads -auth_thread = start_thread(ZvmAuthHandler) -data_thread = start_thread(GetDataFunc) -#stats_thread = start_thread(GetStatsFunc()) -vra_metrics_thread = start_thread(GetVraMetrics) -webserver_thread = start_thread(WebServer) -probe_thread = start_thread(ThreadProbe) - -# loop indefinitely -while True: - # check if any thread has crashed - sleep(10) - if not probe_thread.is_alive(): - # restart the thread - log.error("Probe Thread Died - Restarting") - probe_thread = start_thread(ThreadProbe) - if not auth_thread.is_alive(): - # restart the thread - log.error("Authentication Thread Died - Restarting") - auth_thread = start_thread(ZvmAuthHandler) - if not data_thread.is_alive(): - # restart the thread - log.error("Data Thread Died - Restarting") - data_thread = start_thread(GetDataFunc) - #if not stats_thread.is_alive(): - # # restart the thread - # log.error("Stats Thread Died - Restarting") - # stats_thread = start_thread(GetStatsFunc()) - if not vra_metrics_thread.is_alive(): - # restart the thread - log.error("VRA Metrics Thread Died - Restarting") - vra_metrics_thread = start_thread(GetVraMetrics) - if not webserver_thread.is_alive(): - # restart the thread - log.error("Webserver Thread Died - Restarting") - webserver_thread = start_thread(WebServer) - sleep(api_timeout) \ No newline at end of file diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index a6bc628..2cb941e 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -9,7 +9,7 @@ import threading import socket from pyVim.connect import SmartConnect, Disconnect from pyVmomi import vim -from time import sleep +from time import sleep, time from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.structures import CaseInsensitiveDict from tinydb import TinyDB, Query @@ -18,8 +18,15 @@ from version import VERSION from vmware.vcenter import vcsite from zvma10.zvma import zvmsite from posthog import Posthog - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +global start_time +start_time = time() + +""" +Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually +""" + callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') @@ -28,245 +35,119 @@ client_id = os.environ.get('CLIENT_ID', 'api-script') client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) api_timeout = int(os.environ.get('API_TIMEOUT', 5)) -LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() +LOGLEVEL = os.environ.get('LOGLEVEL', 'DEBUG').upper() +DISABLE_STATS = os.environ.get('DISABLE_STATS', 'FALSE').upper() version = str(VERSION) vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') -# Get the hostname of the machine -container_id = str(socket.gethostname()) -#set log line format including container_id -log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") -log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) -log_handler.setFormatter(log_formatter) - -log = logging.getLogger("Node-Exporter") -log.setLevel(LOGLEVEL) -log.addHandler(log_handler) -log.info(f"Zerto-Node-Exporter - Version {version}") -log.info(f"Log Level: {LOGLEVEL}") -log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) - -# Global Variables -token = "" -siteId = "NotSet" -siteName = "NotSet" -siteZvmVersion = "" -siteVcVersion = "" -siteZvmMajorVersion = "" -siteZvmMinorVersion = "" -siteZvmUpdateVersion = "" -siteZvmPatchVersion = "" -lastStats = CaseInsensitiveDict() - -# Check if vCenter is set, if not disable VRA metrics -is_vcenter_set = True -if vcenter_host == "vcenter.local": - log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") - is_vcenter_set = False -log.debug("vCenter data collection is enabled") -vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug") - -# Authentication Thread which handles authentication and token refresh for ZVM API -def ZvmAuthHandler(): - log.debug("ZVMAuthHandler Thread Started") - expiresIn = 0 - global token - global siteId - global siteName - retries = 0 - while True: - if expiresIn < 30: - h = CaseInsensitiveDict() - h["Content-Type"] = "application/x-www-form-urlencoded" - - d = CaseInsensitiveDict() - d["client_id"] = client_id - d["client_secret"] = client_secret - d["grant_type"] = "client_credentials" - - uri = "https://" + zvm_url + ":" + zvm_port + "/auth/realms/zerto/protocol/openid-connect/token" - delay = 0 - - try: - response = requests.post(url=uri, data=d, headers=h, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - if 'access_token' not in responseJSON or 'expires_in' not in responseJSON: - log.error("Authentication response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - token = str(responseJSON.get('access_token')) - expiresIn = int(responseJSON.get('expires_in')) - log.info("Authentication successful. Token expires in " + str(expiresIn) + " seconds") - - if response.status_code != 200: - log.error("Authentication request failed with status code " + str(response.status_code)) - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - - # Get Site ID and Name - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/localsite" - delay = 0 - try: - log.debug("Getting Site ID and Name") - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - response = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - response.raise_for_status() - except requests.exceptions.RequestException as e: - retries += 1 - delay = 2 ** retries - log.error("Error while sending authentication request: " + str(e) + ". Retrying in " + str(delay) + " seconds") - sleep(delay) - continue - else: - retries = 0 - - responseJSON = response.json() - log.debug(responseJSON) - if 'SiteIdentifier' not in responseJSON or 'SiteName' not in responseJSON: - log.error("LocalSite API response does not contain expected keys") - delay = 2 ** retries - sleep(delay) - retries += 1 - continue - else: - siteId = str(responseJSON.get('SiteIdentifier')) - siteName = str(responseJSON.get('SiteName')) - siteZvmVersion = str(responseJSON.get('Version')) - siteVcVersion = str(responseJSON.get('SiteTypeVersion')) - - # Break out ZVM version strings - siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".") - siteZvmUpdateVersion = siteZvmUpdateVersion[0] - if (len(siteZvmUpdateVersion) > 1): - siteZvmPatchVersion = siteZvmUpdateVersion[1] - else: - siteZvmPatchVersion = "0" - log.info("Site ID: " + siteId + " Site Name: " + siteName) - - expiresIn -= 10 + delay - log.debug("Token Expires in " + str(expiresIn) + " seconds") - sleep(10) - -''' # Thread which gets VM level encryption statistics from ZVM API -def GetStatsFunc(): +def GetStatsFunc(zvm_instance): tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging dbvm = Query() dbvpg = Query() + zvm = zvm_instance while (True) : - global token global siteId global siteName - if (token != ""): - log.info("Got Auth Token!") - log.debug("token: " + str(token)) + if (zvm.is_authenticated()): log.debug("Stats Collector Loop Running") metricsDictionary = {} - - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token ## Statistics API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/" - statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - statsapi_json = statsapi.json() - #log.debug(statsapi_json) + statsapi_json = None + statsapi_json = zvm.vms_statistics() + log.debug(statsapi_json) + vms_encryption_metrics = zvm.encryptiondetection_metrics_vms() - for vm in statsapi_json: - oldvmdata = dict() + if statsapi_json is not None: + for vm in statsapi_json: + vmsiteinfo = zvm.vm(vmidentifier=vm['VmIdentifier'], vpgidentifier=vm['VpgIdentifier']) + if vmsiteinfo['ProtectedSite']['identifier'] == zvm.site_id: + log.debug(f"VM is protected at this site - {vm['VmIdentifier']}") + oldvmdata = dict() + if 'EncryptionMetrics' not in vm: + vm['EncryptionMetrics'] = {} + vm['VmName'] = None - CurrentIops = 0 - CurrentWriteCounterInMBs = 0 - CurrentSyncCounterInMBs = 0 - CurrentNetworkTrafficCounterInMBs = 0 - CurrentEncryptedLBs = 0 - CurrentUnencryptedLBs = 0 - CurrentTotalLBs = 0 - CurrentPercentEncrypted = 0 - VMName = "NA" + CurrentIops = 0 + CurrentWriteCounterInMBs = 0 + CurrentSyncCounterInMBs = 0 + CurrentNetworkTrafficCounterInMBs = 0 + CurrentEncryptedLBs = 0 + CurrentUnencryptedLBs = 0 + CurrentTotalLBs = 0 + CurrentPercentEncrypted = 0 + CurrentTrendChangeLevel = 0 + VMName = "NA" - oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + for vmem in vms_encryption_metrics: + if vmem['Link']['identifier'] == vm['VmIdentifier']: + log.debug(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}") + #print(f"Aligning VM Stats and Encryption Metrics for {vm['VmIdentifier']} - {vmem['Link']['name']}") + vm['EncryptionMetrics']['EncryptedData'] = vmem['EncryptionMetrics']['EncryptedData'] + vm['EncryptionMetrics']['NonEncryptedData'] = vmem['EncryptionMetrics']['NonEncryptedData'] + vm['EncryptionMetrics']['TrendChangeLevel'] = vmem['EncryptionMetrics']['TrendChangeLevel'] + vm['VmName'] = vmem['Link']['name'] - log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) - if (oldvmdata): - log.info(vm['VmIdentifier'] + " Record Found, Updating DB") - log.debug(oldvmdata[0]) - log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) + log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) + oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + if (oldvmdata): + log.info(vm['VmIdentifier'] + " Record Found, Updating DB") + log.debug(oldvmdata[0]) + log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) - log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") - VMName = oldvmdata[0]['VmName'] - log.debug("Current VM " + str(VMName)) - CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) - log.debug("CurrentIops " + str(CurrentIops)) - CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) - log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) - CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) - log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) - CurrentEncryptedLBs = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs']) - log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) - CurrentUnencryptedLBs = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs']) - log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) - CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) - log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) - if CurrentTotalLBs != 0: - CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") + VMName = oldvmdata[0]['VmName'] + log.debug("Current VM " + str(VMName)) + CurrentIops = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter']) + log.debug("CurrentIops " + str(CurrentIops)) + CurrentSyncCounterInMBs = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs']) + log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs)) + CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs']) + log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs)) + CurrentWriteCounterInMBs = abs(vm['WriteCounterInMBs'] - oldvmdata[0]['WriteCounterInMBs']) + log.debug("CurrentWriteCounterInMBs " + str(CurrentWriteCounterInMBs)) + CurrentEncryptedLBs = abs(vm['EncryptionMetrics']['EncryptedData'] - oldvmdata[0]['EncryptionMetrics']['EncryptedData']) + log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs)) + CurrentUnencryptedLBs = abs(vm['EncryptionMetrics']['NonEncryptedData'] - oldvmdata[0]['EncryptionMetrics']['NonEncryptedData']) + log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs)) + CurrentTrendChangeLevel = abs(vm['EncryptionMetrics']['TrendChangeLevel'] - oldvmdata[0]['EncryptionMetrics']['TrendChangeLevel']) + log.debug("CurrentTrendChangeLevel " + str(CurrentTrendChangeLevel)) + CurrentTotalLBs = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs) + log.debug("CurrentTotalLBs " + str(CurrentTotalLBs)) + if CurrentTotalLBs != 0: + CurrentPercentEncrypted = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100) + else: + CurrentPercentEncrypted = 0 + log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + else: + log.info(f"{vm['VmIdentifier']} - {vm['VmName']} - No Record Found, Inserting into DB") + #insert original VM record to tempdb + log.debug(tempdb.insert(vm)) + + # Store Calculated Metrics + metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops + metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs + metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs + metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs + #metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs + #metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs + #metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs + #metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted else: - CurrentPercentEncrypted = 0 - log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted)) + log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") + #print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") + else: + log.debug("No VMS in Stats API") - else: - log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB") - #insert original VM record to tempdb - log.debug(tempdb.insert(vm)) - - # update database with VM name, for easier display in Grafana Legends - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier'] - try: - vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vapi_json = vapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - else: - log.debug("vapi_json: " + str(vapi_json)) - tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier']) - log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName']) - VMName = vapi_json['VmName'] - - # Store Calculated Metrics - metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName) + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops - metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs - metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs - metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs - metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs - metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs - metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs - metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus file_object = open('statsmetrics', 'w') @@ -288,134 +169,112 @@ def GetStatsFunc(): else: log.debug("Waiting 1 second for Auth Token") sleep(1) -''' # Function which retrieves stats from various ZVM APIs and stores them in a metrics file -def GetDataFunc(): +def GetDataFunc(zvm_instance): tempdb = TinyDB(storage=MemoryStorage) dbvm = Query() + zvm = zvm_instance while (True) : - global token global siteId global siteName - if (token != ""): - log.debug("Got Auth Token!") - log.debug("token: " + str(token)) + if (zvm.is_authenticated()): log.info("Data Collector Loop Running") - metricsDictionary = {} - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - - ### VPGs API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vpgs/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - vpg_json = service.json() - #log.debug(vpg_json) - for vpg in vpg_json : - metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] - metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] - metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] - metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] - metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] - metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] - metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] - metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] - metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] - metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] - metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] - metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] - metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] - metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] + vpg_json = None + vpg_json = zvm.vpgs() + if(vpg_json is not None): + log.debug("Got VPG JSON") + for vpg in vpg_json : + metricsDictionary["vpg_storage_used_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["UsedStorageInMB"] + metricsDictionary["vpg_actual_rpo{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ActualRPO"] + metricsDictionary["vpg_throughput_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ThroughputInMB"] + metricsDictionary["vpg_iops{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["IOPs"] + metricsDictionary["vpg_provisioned_storage_in_mb{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ProvisionedStorageInMB"] + metricsDictionary["vpg_vms_count{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["VmsCount"] + metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] + metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] + metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] + metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] + metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] + metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] + metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] + metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] + else: + log.debug("No VPGs Found") ### Datastores APIs - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/datastores/" - service = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - ds_json = service.json() - #log.debug(ds_json) - for ds in ds_json : - - log.debug(f"Processing {ds['DatastoreName']}") - - metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] - metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] - metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] - metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] - metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] - metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] - metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] - metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] + ds_json = None + ds_json = zvm.datastores() + if(ds_json is not None): + log.debug("Got Datastores API") + for ds in ds_json : + log.debug(f"Processing {ds['DatastoreName']}") + metricsDictionary["datastore_vras{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumVRAs"] + metricsDictionary["datastore_incoming_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumIncomingVMs"] + metricsDictionary["datastore_outgoing_vms{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["NumOutgoingVMs"] + metricsDictionary["datastore_usage_capacityinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["CapacityInBytes"] + metricsDictionary["datastore_usage_freeinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["FreeInBytes"] + metricsDictionary["datastore_usage_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["UsedInBytes"] + metricsDictionary["datastore_usage_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Datastore"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_protected_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Protected"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_recovery_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Recovery"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_journal_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Journal"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_scratch_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Scratch"]["ProvisionedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_usedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["UsedInBytes"] + metricsDictionary["datastore_usage_zerto_appliances_provisionedinbytes{datastoreIdentifier=\"" + ds['DatastoreIdentifier'] + "\",DatastoreName=\"" + ds['DatastoreName'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = ds["Stats"]["Usage"]["Zerto"]["Appliances"]["ProvisionedInBytes"] + else: + log.debug("No Datastores Found") ## VMs API log.debug("Getting VMs API") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + scratch_vols = None + scratch_vols = zvm.vms() + if(scratch_vols is not None): + log.debug("Got VMs API") + for vm in scratch_vols: + log.debug("Processing VM: " + str(vm['VmName'])) + log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) - vmapi_json = {} - try: - vmapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL) - vmapi_json = vmapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return + if siteId == vm['ProtectedSite']['identifier']: + log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - log.debug("Got VMs API") - log.debug(vmapi_json) - for vm in vmapi_json : - log.debug("Processing VM: " + str(vm['VmName'])) - log.debug("Checking VM " + vm['VmIdentifier'] + " on Protected Site " + vm['ProtectedSite']['identifier'] + " against " + siteId) + if not isinstance(vm["ActualRPO"], int): + vm["ActualRPO"] = -1 + metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] + metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] + metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] + metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] + metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] + metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] + metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] + metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] + metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] + metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] + metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] + log.debug("Processed VM: " + str(vm['VmName'])) - if siteId == vm['ProtectedSite']['identifier']: - log.debug("Found VM " + vm['VmIdentifier'] + " on Protected Site") - - if not isinstance(vm["ActualRPO"], int): - vm["ActualRPO"] = -1 - metricsDictionary["vm_actualrpo{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ActualRPO"] - metricsDictionary["vm_throughput_in_mb{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["ThroughputInMB"] - metricsDictionary["vm_iops{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["IOPs"] - metricsDictionary["vm_journal_hard_limit{VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + str(siteName) + "\"}"] = vm["JournalHardLimit"]["LimitValue"] - metricsDictionary["vm_journal_warning_limit{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalWarningThreshold"]["LimitValue"] - metricsDictionary["vm_journal_used_storage_mb{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["JournalUsedStorageMb"] - metricsDictionary["vm_outgoing_bandwidth_in_mbps{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["OutgoingBandWidthInMbps"] - metricsDictionary["vm_used_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["UsedStorageInMB"] - metricsDictionary["vm_provisioned_storage_in_MB{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["ProvisionedStorageInMB"] - metricsDictionary["vm_status{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["Status"] - metricsDictionary["vm_substatus{VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",VmRecoveryVRA=\"" + str(vm["RecoveryHostName"]) + "\",VmPriority=\"" + str(vm['Priority']) + "\",SiteIdentifier=\"" + str(siteId) + "\",VpgName=\"" + str(vm['VpgName']) + "\",SiteName=\"" + siteName + "\"}"] = vm["SubStatus"] - log.debug("Processed VM: " + str(vm['VmName'])) - - else: - log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") + else: + log.debug("VM " + vm['VmIdentifier'] + " is protected to this site") + else: + log.debug("No VMs Found") ## Volumes API for Scratch Volumes log.debug("Getting Scratch Volumes") - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=scratch" + scratch_vols = None + scratch_vols = zvm.volumes(volumetype="scratch") - volapi_json = {} - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Scratch Volumes API") - if(bool(volapi_json)): - for volume in volapi_json : + if(scratch_vols is not None): + log.debug("Got Scratch Volumes API") + for volume in scratch_vols: #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. metrickey = "scratch_volume_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\",VpgName=\"" + str(volume['Vpg']['Name']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}" @@ -426,24 +285,17 @@ def GetDataFunc(): percentage_used = (volume["Size"]["UsedInBytes"] / volume["Size"]["ProvisionedInBytes"] * 100) percentage_used = round(percentage_used, 1) #metricsDictionary["scratch_volume_percentage_used{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = percentage_used + else: + log.debug("No Scratch Volumes Found") ## Volumes API for Journal Volumes log.debug("Getting Journal Volumes") + journal_vols = None + journal_vols = zvm.volumes(volumetype="journal") - volapi_json = {} - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/volumes?volumeType=journal" - try: - volapi = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - volapi_json = volapi.json() - except Exception as e: - log.error("Error while sending api request: " + str(e)) - VMName = "Unknown" - return - - log.debug("Got Journal Volumes API") - if(bool(volapi_json)): + if(journal_vols is not None): log.debug("Journal Volumes Exist") - for volume in volapi_json : + for volume in journal_vols : log.debug("Journal Volume: " + volume['ProtectedVm']['Name'] + " Calculating total size...") #metricsDictionary["scratch_volume_provisioned_size_in_bytes{ProtectedVm=\"" + volume['ProtectedVm']['Name'] + "\", ProtectedVmIdentifier=\"" + volume['ProtectedVm']['Identifier'] + "\", OwningVRA=\"" + volume['OwningVm']['Name'] + "\"}"] = volume["Size"]["ProvisionedInBytes"] # Determine the key for a given VM, then see if the key is already in the dictionary, if it is add the next disk to the total. If not, create a new key. @@ -464,6 +316,8 @@ def GetDataFunc(): metricsDictionary[metrickey] = metricsDictionary[metrickey] + 1 else: metricsDictionary[metrickey] = 1 + else: + log.debug("No Journal Volumes Exist") ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus log.debug("Writing metrics to file") @@ -491,101 +345,89 @@ def GetDataFunc(): sleep(1) # get VRA CPU and memory usage from vCenter Server -def GetVraMetrics(): - # set up API endpoint and headers - log.debug("GetVraCpuMemory() called") - metricsDictionary = {} - while True: - vra_names = [] - vras = [] - global token - global siteId - global siteName +def GetVraMetrics(zvm_instance): + log.debug("GetVraMetrics thread started") + try: - log.debug("Checking Token in VRA CPU MEM Collector") - if (token != ""): - log.debug("Auth Token Valid!") - log.debug("token: " + str(token)) - log.info("VRA CPU MEM Collector Running") + metricsDictionary = {} + zvm = zvm_instance + while True: + vra_names = [] + vras = [] + global siteId + global siteName - h2 = CaseInsensitiveDict() - h2["Accept"] = "application/json" - h2["Authorization"] = "Bearer " + token - + log.debug("Checking Token in VRA CPU MEM Collector") + if (zvm.is_authenticated()): + log.info("VRA CPU MEM Collector Running") - ### VRA API - uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vras" - - # make API call to get list of VRAs - try: - response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL) - except Exception as e: - log.error(f"Error connecting to {uri}: {e}") - return - else: - log.debug("Response from GET /v1/vras: %s", response.text) - # parse JSON response and get the name of each VRA - - vras = response.json() - - log.debug("VRA names: %s", vras) - log.debug(type(vras)) - for vra in vras : - # Gather other VRA Metrics from Zerto API into Metrics Diectionary - metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] - metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] - metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] - metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] - metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] - metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] - metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] - metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] - metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - - - log.debug("VRA Name: %s", vra['VraName']) - log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}") - - # get the CPU and memory usage for each VRA - if is_vcenter_set: - log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") - try: - log.debug("Trying to get stats from vc module") - vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + ### VRA API + vras_json = None + vras_json = zvm.vras() + log.debug(vras_json) - # get the CPU usage and memory usage for the VM - cpu_usage_mhz = vradata[0] - memory_usage_mb = vradata[1] + if (vras_json is not None): + log.debug("VRA names: %s", vras_json) + log.debug(type(vras)) + for vra in vras_json : + # Gather other VRA Metrics from Zerto API into Metrics Diectionary + metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"] + metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"] + metricsDictionary["vra_protected_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vms"] + metricsDictionary["vra_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Vpgs"] + metricsDictionary["vra_protected_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["ProtectedCounters"]["Volumes"] + metricsDictionary["vra_recovery_vms{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vms"] + metricsDictionary["vra_recovery_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Vpgs"] + metricsDictionary["vra_recovery_volumes{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["RecoveryCounters"]["Volumes"] + metricsDictionary["vra_self_protected_vpgs{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["SelfProtectedVpgs"] - # print the CPU and memory usage for the VM - log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") - metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz - metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb - except: - log.info(f"No VM found with name {vra['VraName']}") - - ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus - file_object = open('vrametrics', 'w') - txt_object = open('vrametrics.txt', 'w') - for item in metricsDictionary : - file_object.write(item) - file_object.write(" ") - file_object.write(str(metricsDictionary[item])) - file_object.write("\n") - txt_object.write(item) - txt_object.write(" ") - txt_object.write(str(metricsDictionary[item])) - txt_object.write("\n") - - file_object.close() - txt_object.close() + log.debug("VRA Name: %s", vra['VraName']) + log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user}") - # This function will get data every 10 seconds - log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") - sleep(scrape_speed * 2) - else: - log.debug("Waiting 1 second for Auth Token") - sleep(1) + # get the CPU and memory usage for each VRA + if is_vcenter_set: + log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") + try: + log.debug("Trying to get stats from vc module") + vradata = vc_connection.get_cpu_mem_used(vra['VraName']) + + # get the CPU usage and memory usage for the VM + cpu_usage_mhz = vradata[0] + memory_usage_mb = vradata[1] + + # print the CPU and memory usage for the VM + log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB") + metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz + metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb + except: + log.info(f"No VM found with name {vra['VraName']}") + else: + log.debug("No VRAs Found") + + ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus + file_object = open('vrametrics', 'w') + txt_object = open('vrametrics.txt', 'w') + for item in metricsDictionary : + file_object.write(item) + file_object.write(" ") + file_object.write(str(metricsDictionary[item])) + file_object.write("\n") + txt_object.write(item) + txt_object.write(" ") + txt_object.write(str(metricsDictionary[item])) + txt_object.write("\n") + + file_object.close() + txt_object.close() + + # This function will get data every 10 seconds + log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds") + sleep(scrape_speed * 2) + else: + log.debug("Waiting 1 second for Auth Token") + sleep(1) + except Exception as e: + log.error(f"Error in GetVraMetrics: {e}") # function which monitors the threads and restarts them if they die def ThreadProbe(): @@ -594,28 +436,27 @@ def ThreadProbe(): log.debug("Thread Probe Started") metricsDictionary = {} - log.debug("Is Auth Thread Alive") - if auth_thread.is_alive(): - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - else: - metricsDictionary["exporter_thread_status{thread=\"" + "AuthHandler" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - - log.debug("Is Data Thread Alive") + uptime = round((time() - start_time) / 60, 1) + metricsDictionary["exporter_uptime{ExporterInstance=\"" + container_id + "\"}"] = uptime if data_thread.is_alive(): + log.debug("Data Thread Is Alive") metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 else: + log.debug("Data Thread Is NOT Alive") metricsDictionary["exporter_thread_status{thread=\"" + "DataStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - #log.debug("Is Stats Thread Alive") - #if stats_thread.is_alive(): - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 - #else: - # metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 + if stats_thread.is_alive(): + log.debug("Stats Thread Is Alive") + metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 + else: + log.debug("Stats Thread Is NOT Alive") + metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - log.debug("Is VRA Metrics Thread Alive") if vra_metrics_thread.is_alive(): + log.debug("VRA Metrics Thread Is Alive") metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 1 else: + log.debug("VRA Metrics Thread Is NOT Alive") metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 log.debug("Writing Probe data to files") @@ -638,7 +479,6 @@ def ThreadProbe(): log.debug("Probe Thread Going to Sleep") sleep(30) - #----------------run http server on port 9999----------------- def WebServer(): log.info("Web Server Started") @@ -651,17 +491,75 @@ def WebServer(): httpd.serve_forever() def start_thread(target_func): - # start a new thread + log.debug(f"Starting thread for {target_func.__name__}") thread = threading.Thread(target=target_func) + thread.daemon = True thread.start() - # return the thread object + log.debug(f"Thread {target_func.__name__} started") return thread -# start the threads -auth_thread = start_thread(ZvmAuthHandler) -data_thread = start_thread(GetDataFunc) -#stats_thread = start_thread(GetStatsFunc()) -vra_metrics_thread = start_thread(GetVraMetrics) +""" +Main Program Logic +""" + +# Get the hostname of the machine +container_id = str(socket.gethostname()) + +#set log line format including container_id +log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") +log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) +log_handler.setFormatter(log_formatter) +log = logging.getLogger("Node-Exporter") +log.setLevel(LOGLEVEL) +log.addHandler(log_handler) +log.info(f"Zerto-Node-Exporter - Version {version}") +log.info(f"Log Level: {LOGLEVEL}") +log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host: " + zvm_url + "\nZVM Port: " + zvm_port + "\nClient-Id: " + client_id + "\nClient Secret: " + client_secret) + +# Initialize zvmsite instance +zvm_instance = zvmsite( + host=zvm_url, + port=zvm_port, + client_id=client_id, + client_secret=client_secret, + grant_type="client_credentials", + loglevel=LOGLEVEL, + logger=log, + stats=DISABLE_STATS +) +# Start the zvmsite authentication thread +zvm_instance.connect() + +""" +Global Variables used by the program +""" +local_site_info = None +siteId = None +siteName = None + +while(siteId is None): + if zvm_instance.is_authenticated(): + sleep(2) + log.debug("Trying Set Global Vars") + siteId = zvm_instance.site_id + siteName = zvm_instance.site_name + +lastStats = CaseInsensitiveDict() + +# Check if vCenter is set, if not disable VRA metrics +is_vcenter_set = True +if vcenter_host == "vcenter.local": + log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics") + is_vcenter_set = False +log.debug("vCenter data collection is enabled") +vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug", logger=log) + +# Starting threads +vra_metrics_thread = start_thread(lambda: GetVraMetrics(zvm_instance)) +data_thread = start_thread(lambda: GetDataFunc(zvm_instance)) +stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) +log.debug("Starting VRA Metrics") + webserver_thread = start_thread(WebServer) probe_thread = start_thread(ThreadProbe) @@ -673,22 +571,18 @@ while True: # restart the thread log.error("Probe Thread Died - Restarting") probe_thread = start_thread(ThreadProbe) - if not auth_thread.is_alive(): - # restart the thread - log.error("Authentication Thread Died - Restarting") - auth_thread = start_thread(ZvmAuthHandler) if not data_thread.is_alive(): # restart the thread log.error("Data Thread Died - Restarting") - data_thread = start_thread(GetDataFunc) - #if not stats_thread.is_alive(): - # # restart the thread - # log.error("Stats Thread Died - Restarting") - # stats_thread = start_thread(GetStatsFunc()) + data_thread = start_thread(GetDataFunc(zvm_instance)) + if not stats_thread.is_alive(): + # restart the thread + log.error("Stats Thread Died - Restarting") + stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) if not vra_metrics_thread.is_alive(): # restart the thread log.error("VRA Metrics Thread Died - Restarting") - vra_metrics_thread = start_thread(GetVraMetrics) + vra_metrics_thread = start_thread(GetVraMetrics(zvm_instance)) if not webserver_thread.is_alive(): # restart the thread log.error("Webserver Thread Died - Restarting") diff --git a/app/vmware/__init__.py b/app/vmware/__init__.py index a5312ec..97bca9c 100644 --- a/app/vmware/__init__.py +++ b/app/vmware/__init__.py @@ -1,4 +1,4 @@ -print("Initializing zvma10 package...") +print("Initializing vcenter package...") #from .zvma import zvm from .vcenter import vcsite \ No newline at end of file diff --git a/app/vmware/__pycache__/__init__.cpython-310.pyc b/app/vmware/__pycache__/__init__.cpython-310.pyc index ab8f405b2c8e63e188ccd6d8c8f3e82ee59323ea..c2023c5b9da80ed44e5c6717e8b2ce31685c1a3b 100644 GIT binary patch delta 46 zcmaFI_@0qBpO=@50R&bzr^Za=wGox~%*!mvOw7rw%FIhwC`(SwD@iSy7&{RFGz$*? delta 45 zcmaFQ_>Pe`pO=@50SGEeDpI2-^4f^VdFEx7WG3chR%Pa;D^!)`CK?({jGG7mDt``I diff --git a/app/vmware/__pycache__/vcenter.cpython-310.pyc b/app/vmware/__pycache__/vcenter.cpython-310.pyc index 10f076207ab0100f1e9c9148b15f725749a3bd09..736e79879d2452753acd0d20669737694e4c8d7f 100644 GIT binary patch delta 1465 zcmbW1&2Jk;6u@Wp%e(9K#*UpP5v571mUL;8AQFh6O$rSu5rhDVlnPhGvOR<2V0+E% z+HHcZAf;E3qLF$*NcM#z2O>_WR}P4OpnE_!(sSfotyjeLgjQ;GUHQzgjl&wZ~Tyqz=y_ zJi#}zMCbMx>5zieNifnPVlA}8WY6i7ecCm{RL^CUDSf4Dg#%3Om|sGmU#S=%OMI?& z?T+0`cdS081Z*?R?9(?&ahc%KH~oFpJb66p<15<&Sw7B`ZAK@`JEZ ztaJIZGBa!jJY3;zUW@D^W6Q+`t8w8a<)r%jkOA8(TtrUUZ-m8agNs6y$qjXkPRO>p zJc;EAE&~ku@7sovsW^&5&rzE?)TL=^EAm(M8vRiYXyfO~q1My;qzmb98^NWJ^b$}u zxv0H$CBbx?K?FmhUKA`e4xbp0l=LtzzktXgZpc0BLtTu3E`PCZU6=$VvNwvg7GH{kumw7d)V3&!S-D`( z+>TP^W~6F5V8q$0hlln7rSo#h`F(x? zqc{x;0}-zwUPs(QXo%YgJjvn>K%_TCwGqngR5KYT{5G!4%BQL7*qL&<`1<>z8uCKb zZw8x(b@wki^Df%I2lx#?RI!FwM{FSOB0fOm5%&-!`6xZV&Or4>j%xS+LKh#Qw+dnl zQI&7%Q6o6;;&+E0l**b|Dl*kb!fo_8vwP}??+cbdUq$Le4g@W8tv++zF2Yc7d zuHB|`s6=x>LOnzyaY2e)Hzdn>X*h znVsLLgUandG`$FB#u@Cc^(`I~aYX*AE=}SVgku1MKK)uWtJ)MuZR$|Ds7;;z5lceW zNsltfY+d=1JW}>@QW;uj?OL2(xLf6|s2=#S&V>ksYlwmj zjY;i*)fQ{zzur+^(8adv7qpV_AL75P>6kF&306bC`gWA_~5JmM@u zL(CyAAYMaE0pw4)RwhYz5mnQ&WY@=^DVB|k-x2UvRO&%1e1G?6`!70k5$!Jlp5WmS z%ZST}D~PLzYlwFc?;>u>zn$~6;VYn1V`LPtRZTMIs*5_ zy^&2eL-f-{^*Z7sL<7-8_=o_}LVOG;4|^RhaKrkgXm}C#s~b=6l(h2&`uT1#|1v#w zupKITz|$knAf73%#p>bSVf;=$9CYm!R~050mIPik@`u5fZcO03Sw!My2_=Du5N*VP z*F?fYF7VnyAE5&<2^{deIc%!tggKKvos7C!SsE% None: self.stats = stats self.host = host self.port = port @@ -61,7 +63,11 @@ class zvmsite: self._running = False self.LOGLEVEL = loglevel.upper() - self.setup_logging() + if logger is None: + self.setup_logging() + else: + self.log = logger + atexit.register(self.disconnect) self._running = True @@ -74,7 +80,7 @@ class zvmsite: self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded') self.log.debug("Sent PostHog Hook") - def __authhandler__(self): + def __authhandler__(self) -> None: self.log.info(f"Log Level set to {self.LOGLEVEL}") if not self.__connected__: context = ssl.create_default_context() @@ -110,6 +116,10 @@ class zvmsite: self.expiresIn = int(response['expires_in']) self.log.info("Authentication successful") self.__connected__ = True + local_site_info = self.local_site() + self.site_id = local_site_info['SiteIdentifier'] + self.site_name = local_site_info['SiteName'] + else: self.log.error("Authentication failed") sleep(2 ** retries) @@ -131,7 +141,7 @@ class zvmsite: log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S") log_handler = RotatingFileHandler(filename=f"./logs/Log-{container_id}.log", maxBytes=1024*1024*100, backupCount=5) log_handler.setFormatter(log_formatter) - self.log = logging.getLogger("Node-Exporter") + self.log = logging.getLogger("ZVM10 Module") self.log.setLevel(self.LOGLEVEL) self.log.addHandler(log_handler) @@ -322,6 +332,23 @@ class zvmsite: return False # Return False if the try block didn't execute successfully + def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, + entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]: + + params = { + 'startdate': startdate, + 'enddate': enddate, + 'vpgid': vpgid, + 'zorgidentifier': zorgidentifier, + 'level': level, + 'entity': entity, + 'helpidentifier': helpidentifier, + 'isdismissed': isdismissed + } + + uri = self.construct_url("v1/alerts", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + def alert_levels(self) -> List[str]: params = { @@ -345,25 +372,8 @@ class zvmsite: uri = self.construct_url(f"v1/alerts/helpidentifiers", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def alerts(self, startdate=None, enddate=None, vpgid=None, zorgidentifier=None, level=None, - entity=None, helpidentifier=None, isdismissed: bool = None) -> List[Dict[str, Any]]: - - params = { - 'startdate': startdate, - 'enddate': enddate, - 'vpgid': vpgid, - 'zorgidentifier': zorgidentifier, - 'level': level, - 'entity': entity, - 'helpidentifier': helpidentifier, - 'isdismissed': isdismissed - } - - uri = self.construct_url("v1/alerts", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def datastore(self, datastoreidentifier=None): + + def datastore(self, datastoreidentifier=None) -> Dict[str, Any]: if datastoreidentifier is None: self.log.error("Datastore identifier is required for get_datastore function.") @@ -382,6 +392,59 @@ class zvmsite: uri = self.construct_url("v1/datastores", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def datetime_local(self) -> datetime: + params = {} + uri = self.construct_url(f"v1/serverDateTime/serverDateTimeLocal", {}) + response = self.make_api_request("GET", uri, headers=self.apiheader) + + if response is not None: + # Extract the datetime string from the JSON response + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise ValueError(error_message) + + def datetime_utc(self,) -> datetime: + params = {} + uri = self.construct_url(f"v1/serverDateTime/serverDateTimeUtc", params) + response = self.make_api_request("GET", uri, headers=self.apiheader) + + if response is not None: + # Extract the datetime string from the JSON response + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise ValueError(error_message) + + def datetime_check(self, dt_str: str) -> datetime: + try: + # Try to parse the string into a datetime object + dt = parser.isoparse(dt_str) + except ValueError: + # If parsing fails, raise an error + raise ValueError("The 'dt_str' parameter must be a valid datetime string.") + + # Format the datetime object for the API call + formatted_datetime = dt.isoformat() + + params = {'datetime': formatted_datetime} + + # Construct the URL with the datetime argument + uri = self.construct_url(f"v1/serverDateTime/dateTimeArgument", params) + + # Make the API request + response = self.make_api_request("GET", uri, headers=self.apiheader) + + # Check if the response is not None and parse the datetime string + if response is not None: + return parser.isoparse(response) + else: + error_message = "API request failed or returned None" + self.log.error(error_message) + raise requests.exceptions.HTTPError(error_message) def encryptiondetection_enable(self): @@ -450,7 +513,7 @@ class zvmsite: uri = self.construct_url("v1/encryptionDetection/suspected/vpgs", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def event(self, eventidentifier=None): + def event(self, eventidentifier=None) -> Dict[str, Any]: if eventidentifier is None: self.log.error("Event identifier is required for get event function.") @@ -461,30 +524,6 @@ class zvmsite: uri = self.construct_url(f"v1/events/{eventidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_types(self): - - params = { - } - - uri = self.construct_url(f"v1/events/types", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_entities(self): - - params = { - } - - uri = self.construct_url(f"v1/events/entities", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def event_categories(self): - - params = { - } - - uri = self.construct_url(f"v1/events/categories", params) - return self.make_api_request("GET", uri, headers=self.apiheader) def events(self, startdate=None, enddate=None, vpgid=None, sitename=None, zorgidentifier=None, eventtype=None, entitytype=None, category=None, username=None, alertidentifier=None) -> List[Dict[str, Any]]: @@ -504,8 +543,74 @@ class zvmsite: uri = self.construct_url("v1/events", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_types(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/types", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_entities(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/entities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def event_categories(self) -> List[str]: + + params = { + } + + uri = self.construct_url(f"v1/events/categories", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def license(self) -> Dict[str, Any]: + + params = { + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("GET", uri, headers=self.apiheader) - def local_site(self): + def license_delete(self) -> bool: + params = {} + uri = self.construct_url(f"v1/license", params) + + try: + response = self.make_api_request("DELETE", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to delete license: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending license delete request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def license_apply(self, license=None): + if license is None: + self.log.error("A license key is required for apply license function.") + raise ValueError("License key is required.") + + params = { + } + + license = { + "licenseKey": license + } + + uri = self.construct_url(f"v1/license", params) + return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) + + def local_site(self) -> Dict[str, Any]: params = { } @@ -521,15 +626,26 @@ class zvmsite: uri = self.construct_url(f"v1/localsite/pairingstatuses", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def local_site_send_billing(self): - - params = { - } - + def local_site_send_billing(self) -> bool: + params = {} uri = self.construct_url(f"v1/localsite/settings/sendusage", params) - return self.make_api_request("POST", uri, headers=self.apiheader) - - def local_site_banner(self): + + try: + response = self.make_api_request("POST", uri, headers=self.apiheader) + # Check if the response status code is 200 (OK) + if response.status_code == 200: + return True + else: + # Log and raise an exception for any non-200 status codes + self.log.error(f"Failed to send billing information: {response.status_code}") + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.log.error(f"Error while sending billing information request: {e}") + raise + + return False # Return False if the try block didn't execute successfully + + def local_site_banner(self) -> Dict[str, Any]: params = { } @@ -549,40 +665,7 @@ class zvmsite: # uri is spelled incorrectly because it is also spelled incorrectly in zerto uri = self.construct_url(f"v1/localsite/settings/logingbanner", params) return self.make_api_request("PUT", uri, json_data=data, headers=self.apiheader) - - def license(self): - params = { - } - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def license_delete(self): - - params = { - } - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("DELETE", uri, headers=self.apiheader) - - def license_apply(self, license=None): - - if license is None: - self.log.error("A license key is required for apply license function.") - raise ValueError("License key is required.") - - params = { - } - - license = { - "licenseKey": license - } - - - uri = self.construct_url(f"v1/license", params) - return self.make_api_request("PUT", uri, json_data=license, headers=self.apiheader) - def peer_sites(self) -> List[Dict[str, Any]]: params = { @@ -591,7 +674,7 @@ class zvmsite: uri = self.construct_url(f"v1/peersites", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def peer_site(self, siteidentifier=None): + def peer_site(self, siteidentifier=None) -> Dict[str, Any]: if siteidentifier is None: self.log.error("Site identifier is required for get site function.") raise ValueError("Site identifier is required.") @@ -602,7 +685,7 @@ class zvmsite: uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def peer_sites_pairing_statues(self): + def peer_sites_pairing_statues(self) -> List[str]: params = { } @@ -644,12 +727,83 @@ class zvmsite: uri = self.construct_url(f"v1/peersites/{siteidentifier}", params) return self.make_api_request("DELETE", uri, json=data, headers=self.apiheader) - def peer_site_pairing_token(self): + def peer_site_pairing_token(self) -> Dict[str, Any]: params = {} uri = self.construct_url(f"v1/peersites/generatetoken", params) return self.make_api_request("POST", uri, headers=self.apiheader) + def recovery_reports(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, vpgname=None, recoverytype=None, state=None) -> List[Dict[str, Any]]: + + params = { + 'starttime': starttime, + 'endtime': endtime, + 'pagenumber': pagenumber, + 'pagesize': pagesize, + 'vpgname': vpgname, + 'recoverytype': recoverytype, + 'state': state + } + + uri = self.construct_url("v1/reports/recovery", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def recovery_report(self, recoveryoperationidentifier=None) -> Dict[str, Any]: + + if recoveryoperationidentifier is None: + self.log.error("RecoveryOperationIdentifier is required for function.") + raise ValueError("RecoveryOperationIdentifier is required.") + + params = {} + + uri = self.construct_url(f"v1/reports/recovery/{recoveryoperationidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def resources_report(self, starttime=None, endtime=None, pagenumber=None, pagesize=None, zorgname=None, vpgname=None, vmname=None, + protectedsitename=None, protectedclustername=None, protectedhostname=None, protectedorgvdc=None, protectedvcdorg=None, recoverysitename=None, + recoveryclustername=None, recoveryhostname=None, recoveryorgvdc=None, recoveryvcdorg=None) -> List[Dict[str, Any]]: + + params = { + 'starttime': starttime, + 'endtime': endtime, + 'pagenumber': pagenumber, + 'pagesize': pagesize, + 'vpgname': vpgname, + 'vmname': vmname, + 'protectedsitename': protectedsitename, + 'protectedclustername': protectedclustername, + 'protectedhostname': protectedhostname, + 'protectedorgvdc': protectedorgvdc, + 'protectedvcdorg': protectedvcdorg, + 'recoverysitename': recoverysitename, + 'recoveryclustername': recoveryclustername, + 'recoveryhostname': recoveryhostname, + 'recoveryorgvdc': recoveryorgvdc, + 'recoveryvcdorg': recoveryvcdorg + } + + uri = self.construct_url("v1/reports/resources", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def service_profiles(self) -> List[Dict[str, Any]]: + + params = { + } + + uri = self.construct_url(f"/v1/serviceprofiles", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def service_profile(self, serviceProfileIdentifier=None) -> Dict[str, Any]: + if siteidentifier is None: + self.log.error("Service Profile identifier is required for get site function.") + raise ValueError("Service Profile identifier is required.") + + params = { + } + + uri = self.construct_url(f"/v1/serviceprofiles/{serviceProfileIdentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + def tasks(self, startedbeforedate=None, startedafterdate=None, completedbeforedate=None, completedafterdate=None, tasktype=None, status=None) -> List[Dict[str, Any]]: params = { @@ -664,7 +818,7 @@ class zvmsite: uri = self.construct_url("v1/tasks", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def task(self, taskidentifier=None): + def task(self, taskidentifier=None) -> Dict[str, Any]: if taskidentifier is None: self.log.error("Task identifier is required for function.") @@ -675,7 +829,7 @@ class zvmsite: uri = self.construct_url(f"v1/tasks/{taskidentifier}", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def task_types(self): + def task_types(self) -> List[str]: params = { } @@ -683,56 +837,13 @@ class zvmsite: uri = self.construct_url(f"v1/tasks/types", params) return self.make_api_request("GET", uri, headers=self.apiheader) - def vpg(self, vpgidentifier=None): + def vms_statistics(self) -> List[Dict[str, Any]]: - if vpgidentifier is None: - self.log.error("Vpg identifier is required for get_vpg function.") - raise ValueError("VM identifier is required.") - - params = { - } + params = { } - uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params) + uri = self.construct_url("v1/statistics/vms", params) return self.make_api_request("GET", uri, headers=self.apiheader) - - def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, - recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, - zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]: - - params = { - 'vpgid': vpgid, - 'vpgname': vpgname, - 'vpgstatus': vpgstatus, - 'vpgsubstatus': vpgsubstatus, - 'protectedsitetype': protectedsitetype, - 'recoverysitetype': recoverysitetype, - 'protectedsiteidentifier': protectedsiteidentifier, - 'recoverysiteidentifier': recoverysiteidentifier, - 'zorgidentifier': zorgidentifier, - 'priority': priority, - 'serviceprofileidentifier': serviceprofileidentifier - } - - uri = self.construct_url("v1/vpgs", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - - def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True): - if vpgidentifier is None: - self.log.error("VPG identifier is required for delete_vpg function.") - raise ValueError("VPG identifier is required.") - - # URL with vpgidentifier in the path - uri = self.construct_url(f"v1/vpgs/{vpgidentifier}") - - # Data to be sent in the request body - data = { - "keepRecoveryVolumes": keeprecoveryvolumes, - "force": force - } - - # Make the POST request - return self.make_api_request("POST", uri, data=data, headers=self.apiheader) - + def vms(self, vmidentifier=None, vmname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, zorgname=None, priority=None, includebackupvms: bool = None, includemountedvms: bool = None) -> List[Dict[str, Any]]: @@ -773,8 +884,8 @@ class zvmsite: def vm_pointintime(self, vmidentifier=None, vpgidentifier=None, includebackupvms: bool = None, includemountedvms: bool = None): if vmidentifier is None: - self.log.error("VM identifier is required for get_vm function.") - raise ValueError("VM identifier is required.") + self.log.error("VM identifier is required for vm_pointintime function.") + raise ValueError("VM identifier is required for vm_pointintime.") params = { 'vpgidentifier': vpgidentifier, @@ -783,8 +894,36 @@ class zvmsite: } uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsintime", params) - return self.make_api_request("GET", uri, headers=self.apiheader) - + stats = self.make_api_request("GET", uri, headers=self.apiheader) + + if isinstance(stats, list) and not stats: + self.log.error("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.") + raise ValueError("No points in time found for the specified VM. Or the VM is in Multiple VPGs, try specifing vpgidentifier.") + elif stats is None: + self.log.error("VM not found, or vpgidentifier must be specified") + raise ValueError("VM not found, or vpgidentifier must be specified") + else: + return stats + + def vm_pointintime_stats(self, vmidentifier=None, vpgidentifier=None): + + if vmidentifier is None: + self.log.error("VM identifier is required for get_vm function.") + raise ValueError("VM identifier is required.") + + params = { + 'vpgidentifier': vpgidentifier + } + + uri = self.construct_url(f"v1/vms/{vmidentifier}/pointsInTime/stats", params) + stats = self.make_api_request("GET", uri, headers=self.apiheader) + + if stats is None: + self.log.error("VM not found, or vpgidentifier must be specified") + raise ValueError("VM not found, or vpgidentifier must be specified") + else: + return stats + def volumes(self, volumetype=None, vpgidentifier=None, datastoreidentifier=None, protectedvmidentifier=None, owningvmidentifier=None) -> List[Dict[str, Any]]: if volumetype: valid_volumetypes = ["scratch", "journal", "recovery", "protected", "appliance"] @@ -805,8 +944,238 @@ class zvmsite: uri = self.construct_url("v1/volumes", params) return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for get_vpg function.") + raise ValueError("VM identifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_checkpoints(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_take_checkpoint(self, vpgidentifier=None, checkpointname=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + # Construct the JSON payload + json_payload = {"checkpointname": "Checkpoint By Python ZVM Module"} + if checkpointname is not None: + json_payload["checkpointname"] = checkpointname + + params = { } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints", params) + return self.make_api_request("POST", uri, json_data=json_payload, headers=self.apiheader) + + def vpg_checkpoint_stats(self, vpgidentifier=None) -> Dict[str, Any]: + + if vpgidentifier is None: + self.log.error("Vpg identifier is required for vpg_checkpoints function.") + raise ValueError("vpgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}/checkpoints/stats", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpgs(self, vpgid=None, vpgname=None, vpgstatus=None, vpgsubstatus=None, protectedsitetype=None, + recoverysitetype=None, protectedsiteidentifier=None, recoverysiteidentifier=None, + zorgidentifier=None, priority=None, serviceprofileidentifier=None) -> List[Dict[str, Any]]: + + params = { + 'vpgid': vpgid, + 'vpgname': vpgname, + 'vpgstatus': vpgstatus, + 'vpgsubstatus': vpgsubstatus, + 'protectedsitetype': protectedsitetype, + 'recoverysitetype': recoverysitetype, + 'protectedsiteidentifier': protectedsiteidentifier, + 'recoverysiteidentifier': recoverysiteidentifier, + 'zorgidentifier': zorgidentifier, + 'priority': priority, + 'serviceprofileidentifier': serviceprofileidentifier + } + + uri = self.construct_url("v1/vpgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) - def __set_zvm_version__(self): + def vpg_delete(self, vpgidentifier=None, keeprecoveryvolumes=True, force=True): + if vpgidentifier is None: + self.log.error("VPG identifier is required for delete_vpg function.") + raise ValueError("VPG identifier is required.") + + # URL with vpgidentifier in the path + uri = self.construct_url(f"v1/vpgs/{vpgidentifier}") + + # Data to be sent in the request body + data = { + "keepRecoveryVolumes": keeprecoveryvolumes, + "force": force + } + + # Make the POST request + return self.make_api_request("POST", uri, data=data, headers=self.apiheader) + + def vpg_retention_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/retentionpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_priorities(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/priorities", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_entity_types(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/entitytypes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_statuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/statuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_substatuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/substatuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_failover_shutdown_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/failovershutdownpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vpg_failover_commit_policies(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vpgs/failovercommitpolicies", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vras(self, vraname=None, status=None, vraversion=None, hostname=None, ipaddress=None, + vragroup=None, datastorename=None, datastoreclustername=None, networkname=None, vraipconfigurationapi=None) -> List[Dict[str, Any]]: + + params = { + 'vraname': vraname, + 'status': status, + 'vraversion': vraversion, + 'hostname': hostname, + 'ipaddress': ipaddress, + 'vragroup': vragroup, + 'datastorename': datastorename, + 'datastoreclustername': datastoreclustername, + 'networkname': networkname, + 'vraipconfigurationapi': vraipconfigurationapi + } + + uri = self.construct_url(f"v1/vras", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra(self, vraidentifier=None) -> Dict[str, Any]: + + if vraidentifier is None: + self.log.error("vraidentifier is required for vra function.") + raise ValueError("vraidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/{vraidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra_upgrade(self, vraidentifier=None) -> Dict[str, Any]: + + if vraidentifier is None: + self.log.error("vraidentifier is required for vra function.") + raise ValueError("vraidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/{vraidentifier}/upgrade", params) + return self.make_api_request("POST", uri, headers=self.apiheader) + + def vra_statuses(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vras/statuses", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def vra_ipconfigurationtypes(self) -> List[str]: + + params = {} + + uri = self.construct_url(f"v1/vras/ipconfigurationtypes", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + """ + def vra_cluster_settings(self, clusteridentifier=None) -> Dict[str, Any]: + + if clusteridentifier is None: + self.log.error("clusteridentifier is required for vra function.") + raise ValueError("clusteridentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/vras/clusters/{clusteridentifier}/settings", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + """ + + def zorgs(self) -> Dict[str, Any]: + + params = { + } + + uri = self.construct_url(f"v1/zorgs", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def zorg(self, zorgidentifier=None) -> Dict[str, Any]: + + if zorgidentifier is None: + self.log.error("zorgidentifier is required for function.") + raise ValueError("zorgidentifier is required.") + + params = { + } + + uri = self.construct_url(f"v1/zorgs/{zorgidentifier}", params) + return self.make_api_request("GET", uri, headers=self.apiheader) + + def __set_zvm_version__(self) -> None: uri = self.construct_url("v1/localsite") response = self.make_api_request("GET", uri, headers=self.apiheader) if response: @@ -824,7 +1193,7 @@ class zvmsite: self.zvm_version['patch'] = temp[1] if len(temp) > 1 else "0" self.log.info(f"Site ID: {self.site_id}, Site Name: {self.site_name}, Site Type: {self.site_type}") - def version(self): + def version(self) -> Dict[str, Any]: if self.__connected__ and self._running: if self.zvm_version['full'] is None: self.__set_zvm_version__() From a581cdfb6da8092de210df469014d86ff03fc49b Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 21 Dec 2023 10:48:29 -0500 Subject: [PATCH 08/12] Update requirements.txt --- app/requirements.txt | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/app/requirements.txt b/app/requirements.txt index b36e0d6..b112d30 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -1,5 +1,30 @@ -requests -tinydb -tinydb-storage -pyVmomi -pyVim \ No newline at end of file +annotated-types==0.6.0 +async-timeout==4.0.3 +backoff==2.2.1 +boto3==1.28.63 +botocore==1.31.63 +cachetools==5.3.1 +certifi==2023.7.22 +charset-normalizer==3.3.0 +docopt==0.6.2 +idna==3.4 +jmespath==1.0.1 +monotonic==1.6 +posthog==3.0.2 +prompt-toolkit==3.0.39 +pydantic==2.4.2 +pydantic_core==2.10.1 +pyflakes==3.1.0 +Pygments==2.16.1 +python-dateutil==2.8.2 +pyvim==3.0.3 +pyvmomi==8.0.2.0 +redis==5.0.1 +requests==2.31.0 +s3transfer==0.7.0 +six==1.16.0 +tinydb==4.8.0 +tinydb-storage==0.1.0 +typing_extensions==4.8.0 +urllib3==2.0.6 +wcwidth==0.2.8 From 57a18385f3d1a5d7eef24a74ca559055bb4a39d7 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 21 Dec 2023 11:23:53 -0500 Subject: [PATCH 09/12] Update docker-compose.yml --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6d79468..4be15f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.3" services: - custom-exporter-in-python: + zerto-exporter: build: . command: python python-node-exporter.py ports: @@ -11,10 +11,10 @@ services: - ZVM_HOST=192.168.50.60 - ZVM_PORT=443 - CLIENT_ID=api-script - - CLIENT_SECRET=js51tDM8oappYUGRJBhF7bcsedNoHA5j + - CLIENT_SECRET=fcYMFuA5TkIUwp6b3hDUxim0f32z8erk - LOGLEVEL=INFO #Valid settings are CRITICAL, ERROR, WARNING, INFO, DEBUG - VCENTER_HOST=192.168.50.50 - VCENTER_USER=administrator@vsphere.local - - VCENTER_PASSWORD=password + - VCENTER_PASSWORD=Zertodata987! volumes: - "./app:/usr/src/app:rw" From 21f32e5a3134eea07734fc51c73ecc2d4e842ff4 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 21 Dec 2023 12:57:30 -0500 Subject: [PATCH 10/12] updates for docker environments --- Dockerfile | 12 +++++++- app/.pypirc | 3 ++ app/iops.py | 59 ------------------------------------- app/python-node-exporter.py | 10 +++---- docker-compose.yml | 3 +- 5 files changed, 19 insertions(+), 68 deletions(-) create mode 100644 app/.pypirc delete mode 100644 app/iops.py diff --git a/Dockerfile b/Dockerfile index cfcf1fc..ae683c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,16 @@ EXPOSE 9999 WORKDIR /usr/src/app -COPY app/* /usr/src/app/ +# Set PYTHONPATH to include /usr/src/app +ENV PYTHONPATH=/usr/src/app + +# Copy the zerto exporter into the container +COPY app /usr/src/app/ + +# Delete uuid.txt file if it exists +RUN [ -f uuid.txt ] && rm uuid.txt || echo "No uuid.txt file to delete" RUN pip install --no-cache-dir -r requirements.txt + +# Entry point for the container +CMD ["python", "python-node-exporter.py"] \ No newline at end of file diff --git a/app/.pypirc b/app/.pypirc new file mode 100644 index 0000000..3128b30 --- /dev/null +++ b/app/.pypirc @@ -0,0 +1,3 @@ +[pypi] + username = __token__ + password = pypi-AgEIcHlwaS5vcmcCJDZlMmZlNTc4LTI2NTgtNDVlZS04MDA2LTVmMGUzNTQyMzFmZAACKlszLCJjNTQzODQ3Yy1iOGQ1LTQwZDAtOTU5Yy0zNWIxNGVmM2NhNjkiXQAABiBdsEOQWVnk-qd-erTO48YJLKxiztWySeNQ45V6y45fkw diff --git a/app/iops.py b/app/iops.py deleted file mode 100644 index d9d8750..0000000 --- a/app/iops.py +++ /dev/null @@ -1,59 +0,0 @@ -from pyVim.connect import SmartConnect, Disconnect -from pyVmomi import vim, vmodl -import ssl - -# Create an SSL context without certificate verification -context = ssl.create_default_context() -context.check_hostname = False -context.verify_mode = ssl.CERT_NONE - -si = SmartConnect(host='192.168.50.50', - user='administrator@vsphere.local', - pwd='Zertodata987!', - sslContext=context -) - -# Find the virtual machine by name -vm_name = 'Squid' -content = si.RetrieveContent() -vm = None - -for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view: - if obj.name == vm_name: - vm = obj - break - -if vm is None: - print(f"Virtual machine '{vm_name}' not found") - si.Disconnect() - exit(1) - -# Get performance manager -perf_manager = content.perfManager - -# Define the metric ID for write IOPS (counterId = 6) -metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="") - -# Create a real-time query specification -query_spec = vim.PerformanceManager.QuerySpec( - entity=vm, - metricId=[metric_id], - format="normal", -) - -# Query the performance statistics -result = perf_manager.QueryStats(querySpec=[query_spec]) - -if result: - # Get the latest write IOPS value - write_iops = result[0].value[0].value - print(f"Current write IOPS for {vm_name}: {write_iops}") - -# Disconnect from vCenter Server -Disconnect(si) - - - - - - diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index 2cb941e..eddfd64 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -42,8 +42,6 @@ vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50') vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local') vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') - - # Thread which gets VM level encryption statistics from ZVM API def GetStatsFunc(zvm_instance): @@ -137,10 +135,10 @@ def GetStatsFunc(zvm_instance): metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs - #metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs - #metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs - #metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs - #metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs + metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs + metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs + metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted else: log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") #print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") diff --git a/docker-compose.yml b/docker-compose.yml index 4be15f5..f75259b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ version: "3.3" services: zerto-exporter: build: . - command: python python-node-exporter.py ports: - "9999:9999" environment: @@ -17,4 +16,4 @@ services: - VCENTER_USER=administrator@vsphere.local - VCENTER_PASSWORD=Zertodata987! volumes: - - "./app:/usr/src/app:rw" + - "./logs:/usr/src/app/logs/" From 4e95d13c6d1328b99ef161855485679dcb6f86c1 Mon Sep 17 00:00:00 2001 From: Justin Paul Date: Thu, 21 Dec 2023 14:00:10 -0500 Subject: [PATCH 11/12] Update python-node-exporter.py --- app/python-node-exporter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index eddfd64..0f78765 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -139,6 +139,7 @@ def GetStatsFunc(zvm_instance): metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted + metricsDictionary["vm_TrendChangeLevel{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + str(vm['VmName']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTrendChangeLevel else: log.debug(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") #print(f"VM is only recovering to this site, skipping metrics - {vm['VmIdentifier']}") From a0ac6883f4e3413d64f5e2fea7b7549917a1dbf3 Mon Sep 17 00:00:00 2001 From: recklessop Date: Thu, 30 May 2024 20:57:39 -0400 Subject: [PATCH 12/12] updates for zvma10 --- Dockerfile | 23 ++++- app/python-node-exporter.py | 79 ++++++++++++------ app/requirements.txt | 6 +- app/uuid.txt | 1 - app/version.py | 4 +- .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 292 bytes .../__pycache__/vcenter.cpython-311.pyc | Bin 0 -> 10116 bytes app/vmware/vcenter.py | 8 +- app/zvma10/zvma.py | 48 +++++------ 9 files changed, 105 insertions(+), 64 deletions(-) delete mode 100644 app/uuid.txt create mode 100644 app/vmware/__pycache__/__init__.cpython-311.pyc create mode 100644 app/vmware/__pycache__/vcenter.cpython-311.pyc diff --git a/Dockerfile b/Dockerfile index ae683c4..1479756 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,22 @@ -FROM python:3.12-slim +FROM python:3.12.3-slim EXPOSE 9999 +# Install system dependencies +RUN apt-get update \ + && apt-get install -y \ + curl \ + gcc \ + libffi-dev \ + libssl-dev \ + python3-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install Rust and Cargo using curl with IPv4 only +RUN CURL_IPRESOLVE=4 curl https://sh.rustup.rs -sSf | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + WORKDIR /usr/src/app # Set PYTHONPATH to include /usr/src/app @@ -13,7 +28,11 @@ COPY app /usr/src/app/ # Delete uuid.txt file if it exists RUN [ -f uuid.txt ] && rm uuid.txt || echo "No uuid.txt file to delete" +# Install Python dependencies +# Set environment variable for PyO3 compatibility +ENV PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 +RUN pip install --upgrade pip RUN pip install --no-cache-dir -r requirements.txt # Entry point for the container -CMD ["python", "python-node-exporter.py"] \ No newline at end of file +CMD ["python", "python-node-exporter.py"] diff --git a/app/python-node-exporter.py b/app/python-node-exporter.py index 0f78765..9c3a7fd 100644 --- a/app/python-node-exporter.py +++ b/app/python-node-exporter.py @@ -13,7 +13,7 @@ from time import sleep, time from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.structures import CaseInsensitiveDict from tinydb import TinyDB, Query -from tinydbstorage.storage import MemoryStorage +from tinydb.storages import MemoryStorage from version import VERSION from vmware.vcenter import vcsite from zvma10.zvma import zvmsite @@ -27,11 +27,14 @@ start_time = time() Variables: Normally these are imported from the Docker Container, but alternative values can be modified if running the script manually """ +listen_port = int(os.getenv('LISTEN_PORT', 9999)) callhomestats = os.getenv("CALL_HOME_STATS", 'True').lower() in ('false', '0', 'f') verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't') zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60') zvm_port = os.environ.get('ZVM_PORT', '443') -client_id = os.environ.get('CLIENT_ID', 'api-script') +zvm_username = os.environ.get('ZVM_USERNAME', 'admin') +zvm_password = os.environ.get('ZVM_PASSWORD', 'Zertodata987!') +client_id = os.environ.get('CLIENT_ID', 'zerto-client') client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk') scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30)) api_timeout = int(os.environ.get('API_TIMEOUT', 5)) @@ -45,9 +48,11 @@ vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!') # Thread which gets VM level encryption statistics from ZVM API def GetStatsFunc(zvm_instance): - tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') used for storing db on disk for debugging + tempdb = TinyDB(storage=MemoryStorage) # ('./db.json') #(storage=MemoryStorage) used for storing db on disk for debugging dbvm = Query() dbvpg = Query() + dbsite = Query() + zvm = zvm_instance while (True) : global siteId @@ -70,9 +75,11 @@ def GetStatsFunc(zvm_instance): if vmsiteinfo['ProtectedSite']['identifier'] == zvm.site_id: log.debug(f"VM is protected at this site - {vm['VmIdentifier']}") oldvmdata = dict() + # this part of the dictionary will never exist, so not sure why i need this as i set the key/values below in the vmem section. if 'EncryptionMetrics' not in vm: vm['EncryptionMetrics'] = {} vm['VmName'] = None + vm['SiteId'] = zvm.site_id CurrentIops = 0 CurrentWriteCounterInMBs = 0 @@ -95,12 +102,14 @@ def GetStatsFunc(zvm_instance): vm['VmName'] = vmem['Link']['name'] log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier']) - oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']) + oldvmdata = tempdb.search((dbvm.VmIdentifier == vm['VmIdentifier']) & (dbvpg.VpgIdentifier == vm['VpgIdentifier'])) if (oldvmdata): log.info(vm['VmIdentifier'] + " Record Found, Updating DB") - log.debug(oldvmdata[0]) - log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])) - + log.debug("Old Data") + log.debug(oldvmdata) + log.debug(tempdb.update(vm, (dbvm.VmIdentifier == vm['VmIdentifier']) & (dbvpg.VpgIdentifier == vm['VpgIdentifier']))) + log.debug("New Data") + log.debug(vm) log.debug("!@!@!@!@!@ Stats !@!@!@!@!@") VMName = oldvmdata[0]['VmName'] log.debug("Current VM " + str(VMName)) @@ -197,8 +206,12 @@ def GetDataFunc(zvm_instance): metricsDictionary["vpg_configured_rpo_seconds{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["ConfiguredRpoSeconds"] metricsDictionary["vpg_actual_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ActualHistoryInMinutes"] metricsDictionary["vpg_configured_history_in_minutes{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["HistoryStatusApi"]["ConfiguredHistoryInMinutes"] - metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] - metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] + if(vpg["FailSafeHistory"] is None): + metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = 0 + metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = 0 + else: + metricsDictionary["vpg_failsafe_history_in_minutes_actual{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ActualFailSafeHistory"] + metricsDictionary["vpg_failsafe_history_in_minutes_configured{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["FailSafeHistory"]["ConfiguredFailSafeHistory"] metricsDictionary["vpg_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["Status"] metricsDictionary["vpg_substatus{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["SubStatus"] metricsDictionary["vpg_alert_status{VpgIdentifier=\"" + vpg['VpgIdentifier'] + "\",VpgName=\"" + vpg['VpgName'] + "\",VpgPriority=\"" + str(vpg['Priority']) + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vpg["AlertStatus"] @@ -387,9 +400,10 @@ def GetVraMetrics(zvm_instance): if is_vcenter_set: log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs") try: - log.debug("Trying to get stats from vc module") + log.debug("Trying to get stats from vCenter module") vradata = vc_connection.get_cpu_mem_used(vra['VraName']) - + for item in vradata: + log.debug(item) # get the CPU usage and memory usage for the VM cpu_usage_mhz = vradata[0] memory_usage_mb = vradata[1] @@ -399,7 +413,7 @@ def GetVraMetrics(zvm_instance): metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion'] + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb except: - log.info(f"No VM found with name {vra['VraName']}") + log.info(f"No VM found with name {vra['VraName']}, or unexpected response.") else: log.debug("No VRAs Found") @@ -458,7 +472,7 @@ def ThreadProbe(): log.debug("VRA Metrics Thread Is NOT Alive") metricsDictionary["exporter_thread_status{thread=\"" + "VraMetrics" + "\",ExporterInstance=\"" + container_id + "\"}"] = 0 - log.debug("Writing Probe data to files") + log.debug("Writing Thread data to files") file_object = open('threads', 'w') txt_object = open('threads.txt', 'w') for item in metricsDictionary : @@ -471,22 +485,21 @@ def ThreadProbe(): txt_object.write(str(metricsDictionary[item])) txt_object.write("\n") - log.debug("Trying to Close probe txt files") + log.debug("Trying to close Thread txt files") file_object.close() txt_object.close() log.debug("Probe Thread Going to Sleep") sleep(30) -#----------------run http server on port 9999----------------- -def WebServer(): - log.info("Web Server Started") - PORT = 9999 +#----------------run http server on port ----------------- +def WebServer(port): + log.info(f"Web Server Starting on port {port}") Handler = http.server.SimpleHTTPRequestHandler - with socketserver.TCPServer(("", PORT), Handler) as httpd: - log.info(f"Webserver running on port {PORT}") + with socketserver.TCPServer(("", port), Handler) as httpd: + log.info(f"Webserver running on port {port}") httpd.serve_forever() def start_thread(target_func): @@ -519,16 +532,18 @@ log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host zvm_instance = zvmsite( host=zvm_url, port=zvm_port, + username=zvm_username, + password=zvm_password, client_id=client_id, client_secret=client_secret, - grant_type="client_credentials", loglevel=LOGLEVEL, logger=log, stats=DISABLE_STATS ) +# grant_type="client_credentials", # Start the zvmsite authentication thread -zvm_instance.connect() +zvm_instance.connect() """ Global Variables used by the program """ @@ -558,9 +573,9 @@ vra_metrics_thread = start_thread(lambda: GetVraMetrics(zvm_instance)) data_thread = start_thread(lambda: GetDataFunc(zvm_instance)) stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) log.debug("Starting VRA Metrics") - -webserver_thread = start_thread(WebServer) -probe_thread = start_thread(ThreadProbe) +webserver_thread = start_thread(lambda: WebServer(listen_port)) +probe_thread = start_thread(lambda: ThreadProbe) +log.debug(f"ThreadProbe just started on PID {probe_thread}") # loop indefinitely while True: @@ -570,20 +585,30 @@ while True: # restart the thread log.error("Probe Thread Died - Restarting") probe_thread = start_thread(ThreadProbe) + else: + print("Probe Thread is alive") if not data_thread.is_alive(): # restart the thread log.error("Data Thread Died - Restarting") data_thread = start_thread(GetDataFunc(zvm_instance)) + else: + print("Data API Thread is alive") if not stats_thread.is_alive(): # restart the thread log.error("Stats Thread Died - Restarting") stats_thread = start_thread(lambda: GetStatsFunc(zvm_instance)) + else: + print("Stats API Thread is alive") if not vra_metrics_thread.is_alive(): # restart the thread log.error("VRA Metrics Thread Died - Restarting") vra_metrics_thread = start_thread(GetVraMetrics(zvm_instance)) + else: + print("VRA Metrics Thread is alive") if not webserver_thread.is_alive(): # restart the thread log.error("Webserver Thread Died - Restarting") - webserver_thread = start_thread(WebServer) - sleep(api_timeout) \ No newline at end of file + webserver_thread = start_thread(WebServer(listen_port)) + else: + print("WebServer Thread is alive") + sleep(api_timeout) diff --git a/app/requirements.txt b/app/requirements.txt index b112d30..f059f6f 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -12,19 +12,17 @@ jmespath==1.0.1 monotonic==1.6 posthog==3.0.2 prompt-toolkit==3.0.39 -pydantic==2.4.2 -pydantic_core==2.10.1 +pydantic pyflakes==3.1.0 Pygments==2.16.1 python-dateutil==2.8.2 pyvim==3.0.3 pyvmomi==8.0.2.0 redis==5.0.1 -requests==2.31.0 +requests==2.32.0 s3transfer==0.7.0 six==1.16.0 tinydb==4.8.0 -tinydb-storage==0.1.0 typing_extensions==4.8.0 urllib3==2.0.6 wcwidth==0.2.8 diff --git a/app/uuid.txt b/app/uuid.txt deleted file mode 100644 index 6905ced..0000000 --- a/app/uuid.txt +++ /dev/null @@ -1 +0,0 @@ -c50dfc5a-bbd3-49e4-b8f8-688b19e89960 \ No newline at end of file diff --git a/app/version.py b/app/version.py index 3a082fc..7f65076 100644 --- a/app/version.py +++ b/app/version.py @@ -1,5 +1,5 @@ # version.py -VERSION = "2.0.0" +VERSION = "2.1.0" def main(): # Put your main program code here @@ -7,4 +7,4 @@ def main(): if __name__ == '__main__': # This code will be executed only when the file is run as the main program - main() \ No newline at end of file + main() diff --git a/app/vmware/__pycache__/__init__.cpython-311.pyc b/app/vmware/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba84f1c516de211a7ce9f62f5008d9954eff3d4d GIT binary patch literal 292 zcmZ3^%ge<81X%@PX)!?hF^B^L%uvQ>4IpDW0~13kLkhz(Mh1q}Knwwl3@MB$OgW6X zOi@fgHghn8CQFsPXI^GWW@1ieRc2ngLRoTZUP)?^LP273c4B&}o}S)IkakVRTWn>? z#hE3kewxg;SPP0W^Ga^9LlqZ+tX|3R8Dz;XBmIp0+*JLn(&CcLJpHKDqLTc0*NTGt zBA}`Ii3J7vWx3^vMXCDn@tHt-M@DE@T29r23{s9u(Bp8^%0eahAWz(jAoT{>o zJKY15$?mqYUX2s6BHAHZuf#A!pcN1Ekk!5rO=gvctW<5)h*n9KkXCs>+6PBuwGmHy z&aJNMZvSN29qm5t_4TRyd+xdSR-NyjbNjbmubV)aynTL(WeNFL%;>>dNIZ){;uhfu zXNr-yX~GmYPnb=ZH^(e->x4CKo3PQmC1#(nL%Aj9m~haibHYiVt_c@Btuc4pGvP52 zGdV#x+lPd+-!&6*AO5v4!Elyw((ZVQ*>}Xr_=u#s2cu#{OGBMyJ{nhT z^YJ7X8*4YI{xeA_EJYL3!_gRjGMwOIybvj{r4N0OXT6ZUMR+n{;>d)VV?YqogoU$z zNG7bD1E`I20=084pbpLr)X8~(x;QHc=T+_V5iu(9Pyfk8$gpZXH8y;DEaEH~%LZu+ z{6E_bgWe)(Q<{W~>a-Pd&{GR(Gvsv9NZJB9>`6n~2048fV7(!SoN1sryPiwiR~&k} z^m!LD_l*MRZ%9e|yY#BGD zT)xK{m(--!5_nhA&R@fu@tK9QarOs}yD2n zydA7Tan@wUNUv1uJxZTxGGnY*IUf`3MoDjfX)isUc5m#%k#UlCl*)i5?XI!Z;nudW z7?GlJzFj=l+7{#I`B)-c$VpcOKFp2j**GtX;b~}?s%pK^*&6R`<=EDfy{)6Yt>YYRel0~xE%c=+1!MTr+x&ty1qb#8VbIhT-9 z74rjpLgEE>G|A1y`1XKmy^<6q)jFFLB-J@5@`BcVHY|z@Nr6+{^Slt9S`3L|Om#q7 zj3yJRGZey4H55{9bF;I&pgKoRpBNcBKQy9RKql29N`h(^laZ^uqP#T)jWl%ksy`m`=(E|+FVt9VA`${s- zchAg;pn|$5ctJ{rhOXg~@j`cacD8#yz7Q7p?)eB^+^*S0T%l+pDuqI+N==`1=>^DK z1Px1WkbGcA*1F`50W%r&HxhE#TRt*%o(^3z8b zffV9oQqKHbuDhPAyROt-&%TvsY9BK#t4zzvr5w|vFg-HUvsT-v)V9g(M{~8kN^Ng; zaOq^8@hzQv%j6!-k{N75!%As8y%10fzEnfQ{+fJd-uP8Qy`gb#{F`Lx1khuKzAe{%8RUtQ z&>$XyoSq?DYhuuXAYD;k0$y6urlLNU3_Sy*8WKPYiqFWLBN?-+Of$o%hHi$mR~G@& zhW<^P9$4=JhTJ!B1hkJaTWD#dXDapnGls^^SkjgmLziZ(6uXQ(^wzsQ1%HE`XK;^#%tKiN7YL8Xc^Uba0T)WI}WYcv5$W+x;CL|xE~phtrd z(D)J`;`piXTuee;iX&@^%f|7ObJJ=?bCM9mkzGUABK)j`>e~iNHz~B+1>BrM>uZu~7NekJu?OfbkeYTg z@gjZ-nl=>n=^b$DfF|^))KyrD-B2m+*EE}xS#ti$`;k`;9=+G{pc5VtbIxAH*(*DH z^PbA3*JKa77Tl=>59ES}l;EN4iM+q|=IiUk>OGwI1#UWTReVw*Hyz0N4l2HbvhU!N z_O4}z;@i8nxBX5WQ+w9!q&k>AzHWAS-!iSS9S>W6*Z!#Gt4{gc1?AkNoSc{0g&ez} zunWu9o8J7c=G&K)U5AuiKUsF?>spk$j%7!_X)n|%;`3l+uz;ryhmnrc)4ezBd)$oAOri2MFsHWP*ui!ZgDQi1=!Jd9;7;Kr{J$vwP5Q z`ThP~gB6yqDr}H0(~YSAaJQe~2Mp)VlJ5xKeG?p$>5hGFqT4EBbCzxxEjBi*fZ<1= zT152&FjVPZO3TiAp#!oJLzzMb>*)3cB*-kQiP8b`WYh_+`3+X z+a?v8CMzi0rckcTO~h3;=8B;zOFNeL$(YmTA~<9$iRL|o9tn*u+sxRqamE)&!lEfC z;AlV!s@koAByEul^;`-S(PdiG~a~VLDwKA1U zUD#oB+5!CSzYt$wZ)|CMaipzOH`E&dR)nu2C}o`6v@lj9<0_NGfGK_U_ypi6lY?_m z^TU~Tff3^R%uzm1*iopNHb>wvM&C<=)z-UPx|+bxz4@rI^rh5lFd>|$e7p}#?aWx} z*a;pK{_I>R&c{OlKpYzh$71kUh)P%31OQH)#glSZSC_y`b3y_E28y|`6iyAD6&4GC z0|#XG{F#1M1iM(Iok4FKXr+TwGSR_GMxR3I^Vh)`h;hJGVv$V&E!Zg`88zX8 z6VUEet(xs3&c(rFv8cM=n~O%Sj^oO!l}S*ju!94$bLVh|s&9IZ7o|};Gg=}XL>BJz z;n*Bcj}Gl_4R|f{LRk0-R(eocutuo342~O$7vy!!$3s}4n&;!-n}~;!lQXIpUm&`m zq4<@QTB)t0UNTAbdeMH-P}>JvSoY$jY76kau>xHv;y-FQURk}&ee{0z{SvIZnDbpy ze3xY3rMxe=Y)ACStXNkoTja`?l4esm-?ZYJmVMJuc#HXj`MBa{#d5{E-Rhk*4@-Iaa3bT$T5{p~YOFUkUWf zG_F-Qe0n)o-Jw*YjU}Sx@Z~PHzmI&`*KoYSQikeij8IfRLs6aLBNS<2Yu&dI9}Pq_ zuzYJQ<~Cr9i4+mB)D%l^XqT}7x|vE*c*Q_ZT~d5P3&S+h;a6KRRK}hxCSZg zOV2$oQDmD!_5F=+X7Gg!DgH2fU`1|^t` zcqyDfauUff5`H+WJl#b2SZ$kzr3DeihoTK{moNxr<(TA#9 z63GSCg-MQ&NnzFXjy@gLttBZmTL7=&4gql&+>(5vouRlZoI@cf?jpESXtyA$GAN4y zye@5B(O2k%RSeOQ-E?USX}X9Z5mr&P3%m%HyjmUB7BWN^G89eDic}XD!I;%-GPMFx zx7aFGQv$}t;>#ea_(uxG>t>7VvMC>|yBU8R>{$)=C}V=gJoC0UOEnE!4k0C|%izjX7ZtRwHO%9@|}>lFXq$Nt_`fA1GZlw+fRe0yAZ zds03fk@;xOKciqwt`mzZNJ(}eUs=6$?b9QxmAmE2-T9gw%jUeVX4$rGBVdd>2~&~n zU1O@`>fV)UAX>~Z#}ww6%pA)z)!AVH#vM&c?Y=An^3?250{b2ZdR7BHUxdHjEeCpX zfl(zensq^Cph5AsJodM(`rC5;eTsiy)|s!_^&6+c9{S>t%=YEjJ_Tq^U)Bw+!I4n2 z|8dRBt2HkJa_56h%kSp>_4y#14>aU!cdmO$@Rjd~Bk2AfN!Gm{B-M?NtJ+qp+8)07 z@Kw30Em!r1QuRjG4!+cNVt4KMwz*~H{Js7=7w&`}zN54rQCeTiHTNmaeSh#jI{$V5 zR~Nnt$>+|)c|#exlXArncZqbT9+7|-uqO=C!8>}Jm0>HhcS9V+A{__nWYruXPSR186 z{6~P^Xtj0Ua>$iBi}U*L$xzNm1NyS}r{spTI0pBk@Sp1Y9~#)BnmK<~co#+$E&_Rq z?i~SPK)_38VG>CM36Eq7$uyD#5(?&r%E3GasF{c?{wSUcv(Wob19U+^0YoGck~t*v zNEVP>L$ZkEeIzL)X(VVIZvti^gH=CAavjMHB)>rN0g_)L`4EU&Q;xTVxo!iwI}viX z?2uwh(09ugAH!(ke;7b~=|_MV>WV=8ZjQOAFc)P#{@(#{i4X(#7UcQ?E#{a(g&CBY zLGZROA1;AorVNfJzdl6acuWb5WnJI;>;G55IPjxjY;$!!sjOKJFD))z|MWejvRSEo z2@WLQ&b7Av_l6%Hd~oW)=`Z?~jy|PhFxNJuv<>}>0x87HqL#?{<`v((?3>TmcF4YtCzbxJB>W}30A=oY z0A|gK{st1|?s&tmjb3tiXSA?_U+1Q<*+NDeLNcscP}n7o{&K~I7AIWz89Zsu@Nd7s z*qeOn)PAvyrF!*iFV6T;4lD3*;1%Iotb?5dS0j24BXfcPW=L#NzYXKLuGG%r4|%|? zS?W}5w`nE=-i&D;JQS8*PYA~$RdQ#cx{D*HstR9cvFM+%^&}9uy}-MqPX|{$EL^9Q zy!1cIF&7l(g3Mf?2hVSv)ys!(zLs+~E6!%w*{t8F7w*^TeO1Lr6>@D)&UsjI9+ve; zZ$;qB5Eb#T05%~RYj@-QJ{k`23kY{81Q&7eV+u$!q0oEaLDXu5{m_=WC$$+~!0cZj z`70!U4din|xofvHcrPpI|y1B;IxK7GRm*0jLQYCbs$%gk-IYG zoHQM@?N}!UnKJd*0_(&eGbX3)<#n=|XhNcB@cSt5?Be-oT)*?v?jLa*RR>hWlkuo# z?%AP8ID~{E0Um~g25jcTpJRX%yG#GI1^xX8ow$HnwzwkCov&&u+b8@DRN#js{uT&a zqM1xj$RT;Fm?w?$RxwZ9H|SrURNkO}c~WtM{^g1H2K~#E19Ev>w+>oNuqIp6?`Zk= J+tkvD{3kt!3>^Rf literal 0 HcmV?d00001 diff --git a/app/vmware/vcenter.py b/app/vmware/vcenter.py index 92a3f50..5fb928b 100644 --- a/app/vmware/vcenter.py +++ b/app/vmware/vcenter.py @@ -36,7 +36,7 @@ class vcsite: if self.__conn__ is None: context = ssl.create_default_context() if not self.verify_ssl: - log.debug("dont verify SSL") + self.log.debug("dont verify SSL") # Create an SSL context without certificate verification context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -90,7 +90,7 @@ class vcsite: else: self.log.debug(f"{vm_obj.name} is not a VRA") raise ValueError("No VRA Found") - + def get_write_iops(self, vm): try: content = self.__conn__.RetrieveContent() @@ -141,7 +141,7 @@ class vcsite: return average_write_iops else: return None - + def get_average_write_latency(self, vm): try: content = self.__conn__.RetrieveContent() @@ -203,4 +203,4 @@ class vcsite: Disconnect(self.__conn__) self.__conn__ = None self.version = None - self.log.debug(f"Disconnected from vCenter") \ No newline at end of file + self.log.debug(f"Disconnected from vCenter") diff --git a/app/zvma10/zvma.py b/app/zvma10/zvma.py index a199b00..59b5db2 100644 --- a/app/zvma10/zvma.py +++ b/app/zvma10/zvma.py @@ -17,7 +17,7 @@ from dateutil import parser from typing import List, Dict, Tuple, Union, Any, Optional from requests.structures import CaseInsensitiveDict from logging.handlers import RotatingFileHandler -from posthog import Posthog +#from posthog import Posthog import uuid from requests import Request, Session from .version import VERSION @@ -75,10 +75,10 @@ class zvmsite: self.uuid = self.load_or_generate_uuid() # Posthog stats setup - if self.stats: - self.setup_posthog() - self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded') - self.log.debug("Sent PostHog Hook") + #if self.stats: + # self.setup_posthog() + # self.posthog.capture(self.uuid, 'ZVMA10 Python Module Loaded') + # self.log.debug("Sent PostHog Hook") def __authhandler__(self) -> None: self.log.info(f"Log Level set to {self.LOGLEVEL}") @@ -172,10 +172,10 @@ class zvmsite: file.write(new_uuid) return new_uuid - def setup_posthog(self) -> None: - self.posthog = Posthog(project_api_key='phc_HflqUkx9majhzm8DZva8pTwXFRnOn99onA9xPpK5HaQ', host='https://posthog.jpaul.io') - self.posthog.debug = True - self.posthog.identify(distinct_id=self.uuid) + #def setup_posthog(self) -> None: + # self.posthog = Posthog(project_api_key='phc_HflqUkx9majhzm8DZva8pTwXFRnOn99onA9xPpK5HaQ', host='https://posthog.jpaul.io') + # self.posthog.debug = True + # self.posthog.identify(distinct_id=self.uuid) def construct_url(self, path="", params=None) -> str: full_url = f"{self.base_url}/{path}" @@ -236,21 +236,21 @@ class zvmsite: self.log.debug(f'API Request: {method} - {url}') # Posthog stats setup - if self.stats: - temp_base, temp_path = self.deconstruct_url(url) - self.posthog.capture( self.uuid, 'API REQUEST', - { - "url": temp_base, - "port": self.port, - "endpoint": temp_path, - "method": method, - "response_time_ms": int(elapsed_time_ms), - "verify_ssl": self.verify_ssl, - "grant_type": self.grant_type, - "status_code": str(response.status_code), - "sdk_version": self.__version__ - }) - self.log.debug("Sent PostHog Hook") + #if self.stats: + # temp_base, temp_path = self.deconstruct_url(url) + # self.posthog.capture( self.uuid, 'API REQUEST', + # { + # "url": temp_base, + # "port": self.port, + # "endpoint": temp_path, + # "method": method, + # "response_time_ms": int(elapsed_time_ms), + # "verify_ssl": self.verify_ssl, + # "grant_type": self.grant_type, + # "status_code": str(response.status_code), + # "sdk_version": self.__version__ + # }) + # self.log.debug("Sent PostHog Hook") return response.json() except requests.exceptions.RequestException as e: