dev updates

Not finished, but pushing for use on another machine

app/iops.py

@@ -0,0 +1,59 @@
+from pyVim.connect import SmartConnect, Disconnect
+from pyVmomi import vim, vmodl
+import ssl
+
+# Create an SSL context without certificate verification
+context = ssl.create_default_context()
+context.check_hostname = False
+context.verify_mode = ssl.CERT_NONE
+
+si = SmartConnect(host='192.168.50.50', 
+    user='administrator@vsphere.local', 
+    pwd='Zertodata987!', 
+    sslContext=context
+)
+
+# Find the virtual machine by name
+vm_name = 'Squid'
+content = si.RetrieveContent()
+vm = None
+
+for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view:
+    if obj.name == vm_name:
+        vm = obj
+        break
+
+if vm is None:
+    print(f"Virtual machine '{vm_name}' not found")
+    si.Disconnect()
+    exit(1)
+
+# Get performance manager
+perf_manager = content.perfManager
+
+# Define the metric ID for write IOPS (counterId = 6)
+metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="")
+
+# Create a real-time query specification
+query_spec = vim.PerformanceManager.QuerySpec(
+    entity=vm,
+    metricId=[metric_id],
+    format="normal",
+)
+
+# Query the performance statistics
+result = perf_manager.QueryStats(querySpec=[query_spec])
+
+if result:
+    # Get the latest write IOPS value
+    write_iops = result[0].value[0].value
+    print(f"Current write IOPS for {vm_name}: {write_iops}")
+
+# Disconnect from vCenter Server
+Disconnect(si)
+
+
+
+
+
+

app/python-node-exporter.py

@@ -4,31 +4,32 @@ import socketserver
 import os
 import ssl
 import logging
+from logging.handlers import RotatingFileHandler
 import threading
 import socket
 from pyVim.connect import SmartConnect, Disconnect
 from pyVmomi import vim
 from time import sleep
-from logging.handlers import RotatingFileHandler
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 from requests.structures import CaseInsensitiveDict
 from tinydb import TinyDB, Query
 from tinydbstorage.storage import MemoryStorage
 from version import VERSION
+from zvma10.vcenter import vcsite
 
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 verifySSL = os.getenv("VERIFY_SSL", 'False').lower() in ('true', '1', 't')
 zvm_url = os.environ.get('ZVM_HOST', '192.168.50.60')
 zvm_port = os.environ.get('ZVM_PORT', '443')
 client_id = os.environ.get('CLIENT_ID', 'api-script')
-client_secret = os.environ.get('CLIENT_SECRET', 'js51tDM8oappYUGRJBhF7bcsedNoHA5j')
+client_secret = os.environ.get('CLIENT_SECRET', 'fcYMFuA5TkIUwp6b3hDUxim0f32z8erk')
 scrape_speed = int(os.environ.get('SCRAPE_SPEED', 30))
 api_timeout = int(os.environ.get('API_TIMEOUT', 5))
 LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
 version = str(VERSION)
-vcenter_host = os.environ.get('VCENTER_HOST', 'vcenter.local')
+vcenter_host = os.environ.get('VCENTER_HOST', '192.168.50.50')
 vcenter_user = os.environ.get('VCENTER_USER', 'administrator@vsphere.local')
-vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'supersecret')
+vcenter_pwd = os.environ.get('VCENTER_PASSWORD', 'Zertodata987!')
 
 # Get the hostname of the machine
 container_id = str(socket.gethostname())
@@ -50,6 +51,12 @@ log.debug("Running with Variables:\nVerify SSL: " + str(verifySSL) + "\nZVM Host
 token = ""
 siteId = "NotSet"
 siteName = "NotSet"
+siteZvmVersion = ""
+siteVcVersion = ""
+siteZvmMajorVersion = ""
+siteZvmMinorVersion = ""
+siteZvmUpdateVersion = ""
+siteZvmPatchVersion = ""
 lastStats = CaseInsensitiveDict()
 
 # Check if vCenter is set, if not disable VRA metrics
@@ -58,6 +65,7 @@ if vcenter_host == "vcenter.local":
     log.error("vCenter Host not set. Please set the environment variable VCENTER_HOST, turning off VRA CPU and Memory metrics")
     is_vcenter_set = False
 log.debug("vCenter data collection is enabled")
+vc_connection = vcsite(vcenter_host, vcenter_user, vcenter_pwd, loglevel="debug")
 
 # Authentication Thread which handles authentication and token refresh for ZVM API
 def ZvmAuthHandler():
@@ -141,14 +149,25 @@ def ZvmAuthHandler():
             else:
                 siteId = str(responseJSON.get('SiteIdentifier'))
                 siteName = str(responseJSON.get('SiteName'))
+                siteZvmVersion = str(responseJSON.get('Version'))
+                siteVcVersion = str(responseJSON.get('SiteTypeVersion'))
+
+                # Break out ZVM version strings
+                siteZvmMajorVersion, siteZvmMinorVersion, siteZvmUpdateVersion = siteZvmVersion.split(".")
+                siteZvmUpdateVersion = siteZvmUpdateVersion[0]
+                if (len(siteZvmUpdateVersion) > 1):
+                    siteZvmPatchVersion = siteZvmUpdateVersion[1]
+                else:
+                    siteZvmPatchVersion = "0"
                 log.info("Site ID: " + siteId + " Site Name: " + siteName)
                 
         expiresIn -= 10 + delay
         log.debug("Token Expires in " + str(expiresIn) + " seconds")
         sleep(10)
 
-
+'''
 # Thread which gets VM level encryption statistics from ZVM API
+
 def GetStatsFunc():
     tempdb = TinyDB(storage=MemoryStorage) # ('./db.json')   used for storing db on disk for debugging
     dbvm = Query()
@@ -266,6 +285,7 @@ def GetStatsFunc():
         else:
             log.debug("Waiting 1 second for Auth Token")
             sleep(1)
+'''
 
 # Function which retrieves stats from various ZVM APIs and stores them in a metrics file
 def GetDataFunc():
@@ -497,44 +517,17 @@ def GetVraMetrics():
             try:
                 response = requests.get(url=uri, timeout=api_timeout, headers=h2, verify=verifySSL)
             except Exception as e:
-                log.error(f"Error connecting to {endpoint}: {e}")
+                log.error(f"Error connecting to {uri}: {e}")
                 return
             else:
                 log.debug("Response from GET /v1/vras: %s", response.text)
                 # parse JSON response and get the name of each VRA
-                
-                if is_vcenter_set:
-                    # Disable SSL certificate verification
-                    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-                    context.verify_mode = ssl.CERT_NONE
 
-                    # connect to vCenter Server
-                    si = None
-                    try:
-                        si = SmartConnect(host=vcenter_host, user=vcenter_user, pwd=vcenter_pwd, sslContext=context)
-                        log.debug("Connected to vCenter Server %s", vcenter_host)
-                    except Exception as e:
-                        log.error(f"Error connecting to vCenter Server: {e}")
-                        return
-
-                    
-                    # get the root folder of the vCenter Server
-                    content = si.RetrieveContent()
-                    root_folder = content.rootFolder
-
-                    # create a view for all VMs on the vCenter Server
-                    view_manager = content.viewManager
-                    vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True)
-
-                    
-                    
-                    vras = response.json()
+                vras = response.json()
                     
                 log.debug("VRA names: %s", vras)
                 log.debug(type(vras))
-                for vra in vras :
-                    #vra_names.append(vra['VraName'])
-                    
+                for vra in vras :     
                     # Gather other VRA Metrics from Zerto API into Metrics Diectionary
                     metricsDictionary["vra_memory_in_GB{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion']  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["MemoryInGB"]
                     metricsDictionary["vra_vcpu_count{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion']  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = vra["NumOfCpus"]
@@ -548,31 +541,25 @@ def GetVraMetrics():
 
             
                     log.debug("VRA Name: %s", vra['VraName'])
-
+                    log.info(f"vCenter info: T/F = {is_vcenter_set} Host: {vcenter_host} u: {vcenter_user} p: {vcenter_pwd}")
 
                     # get the CPU and memory usage for each VRA
                     if is_vcenter_set:
-                        vm = None
-                        for vm_obj in vm_view.view:
-                            if vm_obj.name == vra['VraName']:
-                                vm = vm_obj
-                                break
-                                
-                        if vm is not None:
-                            log.debug("Found VRA VM in vCenter with name %s", vra['VraName'])
+                        log.debug(f"vCenter Info Is Valid... Trying to get CPU and Memory usage for VRAs")
+                        try:
+                            log.debug("Trying to get stats from vc module")
+                            vradata = vc_connection.get_cpu_mem_used(vra['VraName'])
+                        
                             # get the CPU usage and memory usage for the VM
-                            cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage
-                            memory_usage_mb = vm.summary.quickStats.guestMemoryUsage
+                            cpu_usage_mhz = vradata[0]
+                            memory_usage_mb = vradata[1]
 
                             # print the CPU and memory usage for the VM
-                            log.info(f"VM {vm.name} (name: {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
+                            log.debug(f"VRA {vra['VraName']}) has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
                             metricsDictionary["vra_cpu_usage_mhz{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion']  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = cpu_usage_mhz
                             metricsDictionary["vra_memory_usage_mb{VraIdentifierStr=\"" + vra['VraIdentifierStr'] + "\",VraName=\"" + vra['VraName'] + "\",VraVersion=\"" + vra['VraVersion'] + "\",HostVersion=\"" + vra['HostVersion']  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = memory_usage_mb
-                        else:
+                        except:
                             log.info(f"No VM found with name {vra['VraName']}")
-
-            # Disconnect from vCenter
-            Disconnect(si)
             
             ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
             file_object = open('vrametrics', 'w')
@@ -591,7 +578,7 @@ def GetVraMetrics():
             txt_object.close()
 
             # This function will get data every 10 seconds
-            log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
+            log.debug("Starting Sleep for " + str(int(scrape_speed *2)) + " seconds")
             sleep(scrape_speed * 2)
         else:
             log.debug("Waiting 1 second for Auth Token")
@@ -617,11 +604,11 @@ def ThreadProbe():
         else:
             metricsDictionary["exporter_thread_status{thread=\"" + "DataStats"  + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
 
-        log.debug("Is Stats Thread Alive")
-        if stats_thread.is_alive():
-            metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats"  + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
-        else:
-            metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats"  + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
+        #log.debug("Is Stats Thread Alive")
+        #if stats_thread.is_alive():
+        #    metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats"  + "\",ExporterInstance=\"" + container_id + "\"}"] = 1
+        #else:
+        #    metricsDictionary["exporter_thread_status{thread=\"" + "EncryptionStats"  + "\",ExporterInstance=\"" + container_id + "\"}"] = 0
 
         log.debug("Is VRA Metrics Thread Alive")
         if vra_metrics_thread.is_alive():
@@ -669,10 +656,9 @@ def start_thread(target_func):
     return thread
 
 # start the threads
-
 auth_thread = start_thread(ZvmAuthHandler)
 data_thread = start_thread(GetDataFunc)
-stats_thread = start_thread(GetStatsFunc)
+#stats_thread = start_thread(GetStatsFunc())
 vra_metrics_thread = start_thread(GetVraMetrics)
 webserver_thread = start_thread(WebServer)
 probe_thread = start_thread(ThreadProbe)
@@ -693,10 +679,10 @@ while True:
         # restart the thread
         log.error("Data Thread Died - Restarting")
         data_thread = start_thread(GetDataFunc)
-    if not stats_thread.is_alive():
-        # restart the thread
-        log.error("Stats Thread Died - Restarting")
-        stats_thread = start_thread(GetStatsFunc)
+    #if not stats_thread.is_alive():
+    #    # restart the thread
+    #    log.error("Stats Thread Died - Restarting")
+    #    stats_thread = start_thread(GetStatsFunc())
     if not vra_metrics_thread.is_alive():
         # restart the thread
         log.error("VRA Metrics Thread Died - Restarting")

app/version.py

@@ -1,5 +1,5 @@
 # version.py
-VERSION = "1.2.0"
+VERSION = "1.3.0"
 
 def main():
     # Put your main program code here

app/zvma10/__init__.py

@@ -0,0 +1,4 @@
+print("Initializing zvma10 package...")
+
+#from .zvma import zvm
+from .vcenter import vcsite

app/zvma10/vcenter.py

@@ -0,0 +1,201 @@
+# Class for holding variables related to a site.
+from pyVim.connect import SmartConnect, Disconnect
+from pyVmomi import vim, vmodl
+import ssl
+import datetime
+import logging
+from logging.handlers import RotatingFileHandler
+
+class vcsite:
+    def __init__(self, host, username, password, port=443, verify_ssl=False, loglevel="INFO"):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.verify_ssl = verify_ssl
+        self.version = None
+        self.__conn__ = None
+        self.LOGLEVEL = loglevel.upper()
+
+        #set log line format including container_id
+        log_formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(threadName)s;%(message)s", "%Y-%m-%d %H:%M:%S")
+        log_handler = RotatingFileHandler(filename=f"./logs/Log-Main-vcenter.log", maxBytes=1024*1024*100, backupCount=5)
+        log_handler.setFormatter(log_formatter)
+        self.log = logging.getLogger("Node-Exporter")
+        self.log.setLevel(self.LOGLEVEL)
+        self.log.addHandler(log_handler)
+
+    def connect(self):
+        self.log.info(f"Log Level set to {self.LOGLEVEL}")
+        if self.__conn__ is None:
+            context = ssl.create_default_context()
+            if not self.verify_ssl:
+                print("dont verify SSL")
+                # Create an SSL context without certificate verification
+                context.check_hostname = False
+                context.verify_mode = ssl.CERT_NONE
+
+            # connect to vCenter Server
+            si = None
+            try:
+                self.__conn__ = SmartConnect(host=self.host, user=self.username, pwd=self.password, sslContext=context)
+                about_info = self.__conn__.content.about
+                version = about_info.version
+                self.version = version
+                self.log.debug("Connected to vCenter Server %s", self.host)
+            except Exception as e:
+                self.log.error(f"Error connecting to vCenter Server: {e}")
+
+    def version(self):
+        return self.version
+
+    def get_cpu_mem_used(self, vra):
+            if vra == None:
+                self.log.debug("Get_cpu_mem_used called with no vm name...returning no data")
+                return
+            if self.__conn__ == None:
+                self.log.debug("Trying to get VRA stats without vCenter connection, trying to connect")
+                self.connect()
+
+            # get the root folder of the vCenter Server
+            try:
+                content = self.__conn__.RetrieveContent()
+                root_folder = content.rootFolder
+            except:
+                self.log.debug("Could not get content from vCenter when trying to get VRA stats")
+
+            # create a view for all VMs on the vCenter Server
+            view_manager = content.viewManager
+            vm_view = view_manager.CreateContainerView(root_folder, [vim.VirtualMachine], True)
+
+            vm = None
+            for vm_obj in vm_view.view:
+                if str(vm_obj.name) == str(vra):
+                    vm = vm_obj
+                if vm is not None:
+                    self.log.debug(f"Found VRA VM in vCenter with name {vm.name}")
+                    # get the CPU usage and memory usage for the VM
+                    cpu_usage_mhz = vm.summary.quickStats.overallCpuUsage
+                    memory_usage_mb = vm.summary.quickStats.guestMemoryUsage
+
+                    # print the CPU and memory usage for the VM
+                    self.log.info(f"VM {vm.name} has CPU usage of {cpu_usage_mhz} MHz and memory usage of {memory_usage_mb} MB")
+                    return [cpu_usage_mhz, memory_usage_mb]
+                else:
+                    self.log.debug(f"{vm_obj.name} is not a VRA")
+            raise ValueError("No VRA Found")
+            
+    def get_write_iops(self, vm):
+        try:
+            content = self.__conn__.RetrieveContent()
+        except:
+            self.log.debug("Could not get content from vCenter when trying to get VRA stats")
+
+        # Find the virtual machine by name
+        vm_name = str(vm)
+        vm = None
+
+        for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view:
+            if obj.name == vm_name:
+                vm = obj
+                break
+
+        if vm is None:
+            print(f"Virtual machine '{vm_name}' not found")
+            return
+
+        # Get performance manager
+        perf_manager = content.perfManager
+
+        # Define the metric ID for write IOPS (counterId = 6)
+        metric_id = vim.PerformanceManager.MetricId(counterId=6, instance="")
+
+        # calculate the last 60 seconds
+        end_time = datetime.datetime.now()
+        start_time = end_time - datetime.timedelta(seconds=60)
+
+        # Create a query specification for roll-up data
+        query_spec = vim.PerformanceManager.QuerySpec(
+            entity=vm,
+            metricId=[metric_id],
+            format="normal",
+            startTime=start_time,
+            endTime=end_time,
+            intervalId=20,  # Use an appropriate interval for the roll-up data
+        )
+
+
+        # Query the performance statistics
+        result = perf_manager.QueryStats(querySpec=[query_spec])
+
+        if result:
+            # Get the average write IOPS for the last 60 seconds
+            average_write_iops = sum(result[0].value[0].value) / len(result[0].value[0].value)
+            print(f"Average write IOPS for the last 60 seconds for {vm_name}: {average_write_iops}")
+            return average_write_iops
+        else:
+            return None
+        
+    def get_average_write_latency(self, vm):
+        try:
+            content = self.__conn__.RetrieveContent()
+        except:
+            self.log.debug("Could not get content from vCenter when trying to get VM stats")
+
+        # Find the virtual machine by name
+        vm_name = str(vm)
+        vm = None
+
+        for obj in content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True).view:
+            if obj.name == vm_name:
+                vm = obj
+                break
+
+        if vm is None:
+            self.log.debug(f"Virtual machine '{vm_name}' not found")
+            return None
+
+        # Get performance manager
+        perf_manager = content.perfManager
+
+        # Define the metric ID for write latency (counterId = X) - replace X with the correct counter ID
+        # You'll need to find the specific counter ID for write latency in your vSphere environment.
+        # The counter for write latency may vary based on your configuration.
+
+        metric_id = vim.PerformanceManager.MetricId(counterId=10, instance="")  # Replace X with the correct counter ID
+
+        end_time = datetime.datetime.now()
+        start_time = end_time - datetime.timedelta(seconds=60)
+
+        # Create a query specification for roll-up data
+        query_spec = vim.PerformanceManager.QuerySpec(
+            entity=vm,
+            metricId=[metric_id],
+            format="normal",
+            startTime=start_time,
+            endTime=end_time,
+            intervalId=20,  # Use an appropriate interval for the roll-up data
+        )
+
+        # Query the performance statistics
+        result = perf_manager.QueryStats(querySpec=[query_spec])
+
+        if result:
+            # Get the average write latency for the last 60 seconds
+            if result[0].value[0].value:
+                average_write_latency = sum(result[0].value[0].value) / len(result[0].value[0].value)
+                self.log.info(f"Average write latency for the last 60 seconds for {vm_name}: {average_write_latency}")
+                return average_write_latency
+
+        return None
+
+
+    def disconnect(self):
+        if self.__conn__ == None:
+            self.log.debug(f"vCenter disconnect requested, but not currently connected.")
+            return
+        # Disconnect from vCenter
+        Disconnect(self.__conn__)
+        self.__conn__ = None
+        self.version = None
+        self.log.debug(f"Disconnected from vCenter")

app/zvma10/zvma.py

@@ -0,0 +1,83 @@
+# Class for holding variables related to a site.
+
+class site:
+    def __init__(self):
+        self.zvm_ip = None
+        self.zvm_port = 443
+        self.zvm_client = None
+        self.zvm_secret = None
+        self.zvm_verify_ssl = False
+        self.zvm_token = None
+        self.id = None
+        self.name = None
+        self.zvm_version = None
+        self.zvm_version_major = None
+        self.zvm_version_minor = None
+        self.zvm_version_update = None
+        self.zvm_version_patch = None
+
+        self.vc_ip = None
+        self.vc_port = 443
+        self.vc_username = None
+        self.vc_password = None
+        self.vc_verify_ssl = False
+        self.vc_version = None
+
+
+    # ZVM Related Set / Get Functions
+    def set_zvm_ip(self, value):
+        self.zvm_ip = value
+
+    def set_zvm_port(self, value):
+        self.zvm_port = value
+
+    def set_zvm_client(self, value):
+        self.client = value
+
+    def set_zvm_secret(self, value):
+        self.zvm_secret = value
+
+    def set_zvm_verify_ssl(self, value):
+        self.zvm_verify_ssl = value
+
+    def set_zvm_token(self, value):
+        self.set_zvm_token = value
+
+    def set_zvm_id(self, value):
+        self.id = value
+
+    def set_zvm_name(self, value):
+        self.zvm_name = value
+
+    def set_zvm_version(self, value):
+        # Set main zvm version variable
+        self.zvm_version = value
+
+        # Break out ZVM version string into Major, Minor, Update, Patch variables
+        self.zvm_version_major, self.zvm_version_minor, temp = self.zvm_version.split(".")
+        self.zvm_version_update = temp[0]
+        if (len(temp) > 1):
+            self.zvm_version_patch = temp[1]
+        else:
+            self.zvm_version_patch = "0"
+
+    def set_zvm_token(self, value):
+        self.set_zvm_token = value
+
+    def get_zvm_ip(self):
+        return self.zvm_ip
+
+    def get_zvm_port(self):
+        return self.zvm_port
+    
+    def get_zvm_username(self):
+        return self.zvm_username
+    
+    def get_zvm_password(self):
+        return self.zvm_password
+    
+    def get_zvm_token(self):
+        return self.zvm_token
+    
+
+    # vCenter related Get / Set Functions

app/zvma9_7/GetStatsFunc.py

@@ -0,0 +1,126 @@
+
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+from requests.structures import CaseInsensitiveDict
+from tinydb import TinyDB, Query
+from tinydbstorage.storage import MemoryStorage
+from logging.handlers import RotatingFileHandler
+
+# Function to get VM Encryption Data from ZVMa version 9.7
+def GetStatsFunc():
+    tempdb = TinyDB(storage=MemoryStorage) # ('./db.json')   used for storing db on disk for debugging
+    dbvm = Query()
+    dbvpg = Query()
+    while (True) :
+        global token
+        global siteId
+        global siteName
+
+        if (token != ""):
+            log.info("Got Auth Token!")
+            log.debug("token: " + str(token))
+            log.debug("Stats Collector Loop Running")
+            
+            metricsDictionary = {}
+
+            h2 = CaseInsensitiveDict()
+            h2["Accept"] = "application/json"
+            h2["Authorization"] = "Bearer " + token
+            
+            ## Statistics API
+            uri = "https://" + zvm_url + ":" + zvm_port + "/v1/statistics/vms/"
+            statsapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
+            statsapi_json  = statsapi.json()
+            #log.debug(statsapi_json)
+
+            for vm in statsapi_json:
+                oldvmdata = dict()
+
+                CurrentIops                       = 0
+                CurrentWriteCounterInMBs          = 0
+                CurrentSyncCounterInMBs           = 0
+                CurrentNetworkTrafficCounterInMBs = 0
+                CurrentEncryptedLBs               = 0
+                CurrentUnencryptedLBs             = 0
+                CurrentTotalLBs                   = 0
+                CurrentPercentEncrypted           = 0
+                VMName                            = "NA"
+
+                oldvmdata = tempdb.search(dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier'])
+
+                log.info("Checking TempDB for VM " + vm['VmIdentifier'] + " in VPG " + vm['VpgIdentifier'])
+                if (oldvmdata):
+                    log.info(vm['VmIdentifier'] + " Record Found, Updating DB")
+                    log.debug(oldvmdata[0])
+                    log.debug(tempdb.update(vm, dbvm.VmIdentifier == vm['VmIdentifier'] and dbvpg.VpgIdentifier == vm['VpgIdentifier']))
+
+                    log.debug("!@!@!@!@!@  Stats  !@!@!@!@!@")
+                    VMName                            = oldvmdata[0]['VmName']
+                    log.debug("Current VM " + str(VMName))
+                    CurrentIops                       = abs(vm['IoOperationsCounter'] - oldvmdata[0]['IoOperationsCounter'])
+                    log.debug("CurrentIops " + str(CurrentIops))
+                    CurrentSyncCounterInMBs           = abs(vm['SyncCounterInMBs'] - oldvmdata[0]['SyncCounterInMBs'])
+                    log.debug("CurrentSyncCounterInMBs " + str(CurrentSyncCounterInMBs))
+                    CurrentNetworkTrafficCounterInMBs = abs(vm['NetworkTrafficCounterInMBs'] - oldvmdata[0]['NetworkTrafficCounterInMBs'])
+                    log.debug("CurrentNetworkTrafficCounterInMBs " + str(CurrentNetworkTrafficCounterInMBs))
+                    CurrentEncryptedLBs               = abs(vm['EncryptionStatistics']['EncryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['EncryptedDataInLBs'])
+                    log.debug("CurrentEncryptedLBs " + str(CurrentEncryptedLBs))
+                    CurrentUnencryptedLBs             = abs(vm['EncryptionStatistics']['UnencryptedDataInLBs'] - oldvmdata[0]['EncryptionStatistics']['UnencryptedDataInLBs'])
+                    log.debug("CurrentUnencryptedLBs " + str(CurrentUnencryptedLBs))
+                    CurrentTotalLBs                   = abs(CurrentEncryptedLBs + CurrentUnencryptedLBs)
+                    log.debug("CurrentTotalLBs " + str(CurrentTotalLBs))
+                    if CurrentTotalLBs != 0:
+                        CurrentPercentEncrypted       = ((CurrentEncryptedLBs / CurrentTotalLBs) * 100)
+                    else:
+                        CurrentPercentEncrypted       = 0
+                    log.debug("CurrentPercentEncrypted " + str(CurrentPercentEncrypted))
+
+                else:
+                    log.info(vm['VmIdentifier'] + " No Record Found, Inserting into DB")
+                    #insert original VM record to tempdb
+                    log.debug(tempdb.insert(vm))
+
+                    # update database with VM name, for easier display in Grafana Legends
+                    uri = "https://" + zvm_url + ":" + zvm_port + "/v1/vms/" + vm['VmIdentifier'] +"?vpgIdentifier=" + vm['VpgIdentifier']
+                    try:
+                        vapi = requests.get(url=uri, timeout=3, headers=h2, verify=verifySSL)
+                        vapi_json  = vapi.json()
+                    except Exception as e:
+                        log.error("Error while sending api request: " + str(e))
+                        VMName = "Unknown"
+                    else:
+                        log.debug("vapi_json: " + str(vapi_json))
+                        tempdb.update({'VmName': vapi_json['VmName']}, dbvm.VmIdentifier == vm['VmIdentifier'])
+                        log.info("Added vm to tempdb " + vm['VmIdentifier'] + " - " + vapi_json['VmName'])
+                        VMName = vapi_json['VmName']
+
+                # Store Calculated Metrics
+                metricsDictionary["vm_IoOperationsCounter{VpgIdentifier=\"" + str(vm['VpgIdentifier']) + "\",VmIdentifier=\"" + str(vm['VmIdentifier']) + "\",VmName=\"" + str(VMName)  + "\",SiteIdentifier=\"" + str(siteId) + "\",SiteName=\"" + str(siteName) + "\"}"] = CurrentIops
+                metricsDictionary["vm_WriteCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentWriteCounterInMBs
+                metricsDictionary["vm_SyncCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentSyncCounterInMBs
+                metricsDictionary["vm_NetworkTrafficCounterInMBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentNetworkTrafficCounterInMBs
+                metricsDictionary["vm_EncryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentEncryptedLBs
+                metricsDictionary["vm_UnencryptedDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentUnencryptedLBs
+                metricsDictionary["vm_TotalDataInLBs{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentTotalLBs
+                metricsDictionary["vm_PercentEncrypted{VpgIdentifier=\"" + vm['VpgIdentifier'] + "\",VmIdentifier=\"" + vm['VmIdentifier'] + "\",VmName=\"" + VMName  + "\",SiteIdentifier=\"" + siteId + "\",SiteName=\"" + siteName + "\"}"] = CurrentPercentEncrypted
+
+            ## Write metrics to a human readable metrics.txt file as well as a metrics file that is easy to get in prometheus
+            file_object = open('statsmetrics', 'w')
+            txt_object = open('statsmetrics.txt', 'w')
+            for item in metricsDictionary :
+                file_object.write(item)
+                file_object.write(" ")
+                file_object.write(str(metricsDictionary[item]))
+                file_object.write("\n")
+                txt_object.write(item)
+                txt_object.write(" ")
+                txt_object.write(str(metricsDictionary[item]))
+                txt_object.write("\n")
+            file_object.close()
+            txt_object.close()
+
+            log.debug("Starting Sleep for " + str(scrape_speed) + " seconds")
+            sleep(scrape_speed)
+        else:
+            log.debug("Waiting 1 second for Auth Token")
+            sleep(1)

app/zvma9_7/__init__.py

@@ -0,0 +1,3 @@
+print("Initializing zvma9_7 package...")
+
+from .GetStatsFunc import GetStatsFunc