Comments (17)
timotheeguerin
commented on May 20, 2024
1
I merged this fix, It works for me on dsvm, let me know how it goes.
Thanks.
from batch-insights.
jeroenwo
commented on May 20, 2024
1
Woohoo! Success! Thanks!
from batch-insights.
timotheeguerin
commented on May 20, 2024
Sounds like there is an issue with this code, I'll fix it.
from batch-insights.
jeroenwo
commented on May 20, 2024
Thanks a lot for the effort! But still no luck.. :'(
This is the downloaded nodestats.py
"""TVM stats"""
# stdlib imports
import logging
from datetime import datetime
import os
import time
import platform
from collections import namedtuple
import sys
# non-stdlib imports
import psutil
from applicationinsights import TelemetryClient
VERSION = "0.0.1.1"
_DEFAULT_STATS_UPDATE_INTERVAL = 5
def setup_logger():
# logger defines
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s.%(msecs)03dZ %(levelname)s %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
logger = setup_logger()
# global defines
_IS_PLATFORM_WINDOWS = platform.system() == 'Windows'
def python_environment(): # pragma: no cover
"""
Returns the current python environment information
"""
return ' '.join(
[platform.python_implementation(),
platform.python_version()])
def os_environment():
"""
Get the OS environment
"""
return platform.platform()
def is_windows():
"""
:returns: If running on windows
"""
return _IS_PLATFORM_WINDOWS
def avg(list):
"""
Compute the average of a list
"""
return sum(list) / float(len(list))
def pretty_nb(num, suffix=''):
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1000.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1000.0
return "%.1f%s%s" % (num, 'Yi', suffix)
NodeIOStats = namedtuple('NodeIOStats', ['read_bps', 'write_bps'])
class NodeStats:
"""Persistent Task Stats class"""
def __init__(self,
num_connected_users=0,
num_pids=0,
cpu_count=0,
cpu_percent=None,
mem_total=0,
mem_avail=0,
swap_total=0,
swap_avail=0,
disk_total=0,
disk=None,
net=None):
"""
Map the attributes
"""
self.num_connected_users = num_connected_users
self.num_pids = num_pids
self.cpu_count = cpu_count
self.cpu_percent = cpu_percent
self.mem_total = mem_total
self.mem_avail = mem_avail
self.swap_total = swap_total
self.swap_avail = swap_avail
self.disk_total = disk_total
self.disk = disk or NodeIOStats()
self.net = net or NodeIOStats()
@property
def mem_used(self):
"""
Return the memory used
"""
return self.mem_total - self.mem_avail
class IOThroughputAggregator:
def __init__(self):
self.last_timestamp = None
self.last_read = 0
self.last_write = 0
def aggregate(self, cur_read, cur_write):
"""
Aggregate with the new values
"""
now = datetime.now()
read_bps = 0
write_bps = 0
if self.last_timestamp:
delta = (now - self.last_timestamp).total_seconds()
read_bps = (cur_read - self.last_read) / delta
write_bps = (cur_write - self.last_write) / delta
self.last_timestamp = now
self.last_read = cur_read
self.last_write = cur_write
return NodeIOStats(read_bps, write_bps)
class NodeStatsCollector:
"""
Node Stats Manager class
"""
def __init__(self,
pool_id,
node_id,
refresh_interval=_DEFAULT_STATS_UPDATE_INTERVAL,
app_insights_key=None):
self.pool_id = pool_id
self.node_id = node_id
self.telemetry_client = None
self.first_collect = True
self.refresh_interval = refresh_interval
self.disk = IOThroughputAggregator()
self.network = IOThroughputAggregator()
if app_insights_key or 'APP_INSIGHTS_INSTRUMENTATION_KEY' in os.environ or 'APP_INSIGHTS_KEY' in os.environ:
key = (app_insights_key
or os.environ.get('APP_INSIGHTS_INSTRUMENTATION_KEY')
or os.environ.get('APP_INSIGHTS_KEY'))
logger.info("Detected instrumentation key '%s'. Will upload stats to app insights", key)
self.telemetry_client = TelemetryClient(key)
context = self.telemetry_client.context
context.application.id = 'AzureBatchInsights'
context.application.ver = VERSION
context.device.model = "BatchNode"
context.device.role_name = self.pool_id
context.device.role_instance = self.node_id
else:
logger.info("No instrumentation key detected. Cannot upload to app insights." +
"Make sure you have the APP_INSIGHTS_INSTRUMENTATION_KEY environment variable setup")
def init(self):
"""
Initialize the monitoring
"""
# start cpu utilization monitoring, first value is ignored
psutil.cpu_percent(interval=None, percpu=True)
def _get_network_usage(self):
netio = psutil.net_io_counters()
return self.network.aggregate(netio.bytes_recv, netio.bytes_sent)
def _get_disk_usage(self):
diskio = psutil.disk_io_counters()
return self.disk.aggregate(diskio.read_bytes, diskio.write_bytes)
def _sample_stats(self):
# get system-wide counters
mem = psutil.virtual_memory()
disk_stats = self._get_disk_usage()
net_stats = self._get_network_usage()
swap_total, _, swap_avail, _, _, _ = psutil.swap_memory()
stats = NodeStats(
cpu_count=psutil.cpu_count(),
cpu_percent=psutil.cpu_percent(interval=None, percpu=True),
num_pids=len(psutil.pids()),
# Memory
mem_total=mem.total,
mem_avail=mem.available,
swap_total=swap_total,
swap_avail=swap_avail,
# Disk IO
disk=disk_stats,
# Net transfer
net=net_stats,
)
del mem
return stats
def _collect_stats(self):
"""
Collect the stats and then send to app insights
"""
# collect stats
stats = self._sample_stats()
if self.first_collect:
self.first_collect = False
return
if stats is None:
logger.error("Could not sample node stats")
return
if self.telemetry_client:
self._send_stats(stats)
else:
self._log_stats(stats)
def _send_stats(self, stats):
"""
Retrieve the current stats and send to app insights
"""
process = psutil.Process(os.getpid())
logger.debug("Uploading stats. Mem of this script: %d vs total: %d",
process.memory_info().rss, stats.mem_avail)
client = self.telemetry_client
for cpu_n in range(0, stats.cpu_count):
client.track_metric(
"Cpu usage",
stats.cpu_percent[cpu_n],
properties={
"Cpu #": cpu_n
})
client.track_metric("Memory used", stats.mem_used)
client.track_metric("Memory available", stats.mem_avail)
client.track_metric("Disk read", stats.disk.read_bps)
client.track_metric("Disk write", stats.disk.write_bps)
client.track_metric("Network read", stats.net.read_bps)
client.track_metric("Network write", stats.net.write_bps)
self.telemetry_client.flush()
def _log_stats(self, stats):
logger.info(
"========================= Stats =========================")
logger.info("Cpu percent: %d%% %s", avg(stats.cpu_percent),
stats.cpu_percent)
logger.info("Memory used: %sB / %sB", pretty_nb(stats.mem_used),
pretty_nb(stats.mem_total))
logger.info("Swap used: %sB / %sB", pretty_nb(
stats.swap_avail), pretty_nb(stats.swap_total))
logger.info("Net read: %sBs",
pretty_nb(stats.net.read_bps))
logger.info("Net write: %sBs",
pretty_nb(stats.net.write_bps))
logger.info("Disk read: %sBs",
pretty_nb(stats.disk.read_bps))
logger.info("Disk write: %sBs",
pretty_nb(stats.disk.write_bps))
logger.info("-------------------------------------")
logger.info("")
def run(self):
"""
Start collecting information of the system.
"""
logger.debug("Start collecting stats for pool=%s node=%s",
self.pool_id, self.node_id)
while True:
self._collect_stats()
time.sleep(self.refresh_interval)
def main():
"""
Main entry point for prism
"""
# log basic info
logger.info("Python args: %s", sys.argv)
logger.info("Python interpreter: %s", python_environment())
logger.info("Operating system: %s", os_environment())
logger.info("Cpu count: %s", psutil.cpu_count())
pool_id = os.environ.get('AZ_BATCH_POOL_ID', '_test-pool-1')
node_id = os.environ.get('AZ_BATCH_NODE_ID', '_test-node-1')
# get and set event loop mode
logger.info('enabling event loop debug mode')
app_insights_key = None
if len(sys.argv) > 2:
pool_id = sys.argv[1]
node_id = sys.argv[2]
if len(sys.argv) > 3:
app_insights_key = sys.argv[3]
# create node stats manager
collector = NodeStatsCollector(
pool_id, node_id, app_insights_key=app_insights_key)
collector.init()
collector.run()
if __name__ == '__main__':
main()from batch-insights.
timotheeguerin
commented on May 20, 2024
HHm could you also share the content of node-stats.log too
It should look like this
2018-03-26 14:47:03,497.497Z INFO Python args: ['nodestats.py']
2018-03-26 14:47:03,497.497Z INFO Python interpreter: CPython 2.7.6
2018-03-26 14:47:03,501.501Z INFO Operating system: Linux-4.4.0-116-generic-x86_64-with-Ubuntu-14.04-trusty
2018-03-26 14:47:03,501.501Z INFO Cpu count: 2
2018-03-26 14:47:03,501.501Z INFO enabling event loop debug mode
2018-03-26 14:47:03,501.501Z INFO Detected instrumentation key '[INST_KEY_REMOVED]'. Will upload stats to app insights
2018-03-26 14:47:03,502.502Z DEBUG Start collecting stats for pool=insights-ubuntu node=tvm-57200098_6-20180323t160643z-p
2018-03-26 14:47:08,511.511Z DEBUG Uploading stats. Mem of this script: 15106048 vs total: 3154907136
2018-03-26 14:47:16,206.206Z DEBUG Uploading stats. Mem of this script: 15806464 vs total: 3152560128
2018-03-26 14:47:22,132.132Z DEBUG Uploading stats. Mem of this script: 15867904 vs total: 3152637952
2018-03-26 14:47:28,348.348Z DEBUG Uploading stats. Mem of this script: 15892480 vs total: 3152330752
2018-03-26 14:47:34,522.522Z DEBUG Uploading stats. Mem of this script: 15900672 vs total: 3152723968
2018-03-26 14:47:40,814.814Z DEBUG Uploading stats. Mem of this script: 15912960 vs total: 3152211968
2018-03-26 14:47:47,105.105Z DEBUG Uploading stats. Mem of this script: 15917056 vs total: 3152232448
2018-03-26 14:47:53,255.255Z DEBUG Uploading stats. Mem of this script: 15921152 vs total: 3152146432
2018-03-26 14:47:59,357.357Z DEBUG Uploading stats. Mem of this script: 15925248 vs total: 3152433152
2018-03-26 14:48:09,238.238Z DEBUG Uploading stats. Mem of this script: 15925248 vs total: 3152216064
2018-03-26 14:48:15,757.757Z DEBUG Uploading stats. Mem of this script: 15925248 vs total: 3152384000
2018-03-26 14:48:22,415.415Z DEBUG Uploading stats. Mem of this script: 15925248 vs total: 3152211968
2018-03-26 14:48:28,954.954Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3152449536
2018-03-26 14:48:34,889.889Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3152392192
2018-03-26 14:48:41,096.096Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3150254080
2018-03-26 14:48:47,303.303Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3151904768
2018-03-26 14:48:53,448.448Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3151982592
2018-03-26 14:48:59,667.667Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3152187392
2018-03-26 14:49:11,337.337Z DEBUG Uploading stats. Mem of this script: 15933440 vs total: 3152584704
2018-03-26 14:49:18,614.614Z DEBUG Uploading stats. Mem of this script: 15941632 vs total: 3152560128
2018-03-26 14:49:24,783.783Z DEBUG Uploading stats. Mem of this script: 15945728 vs total: 3152150528
2018-03-26 14:49:30,901.901Z DEBUG Uploading stats. Mem of this script: 15949824 vs total: 3152089088
2018-03-26 14:49:37,092.092Z DEBUG Uploading stats. Mem of this script: 15949824 vs total: 3152039936
2018-03-26 14:49:43,223.223Z DEBUG Uploading stats. Mem of this script: 15953920 vs total: 3151745024
2018-03-26 14:49:49,453.453Z DEBUG Uploading stats. Mem of this script: 15958016 vs total: 3151687680
2018-03-26 14:49:55,670.670Z DEBUG Uploading stats. Mem of this script: 15962112 vs total: 3149443072
Note:There is the instrumentation key displayed in the logs at the top I just noticed so you might not want to share it
I should probably not log that.
from batch-insights.
jeroenwo
commented on May 20, 2024
What if it's not there? Is that part of the problem? Maybe the script didn't run? I am using a Windows DSVM, you have a Linux VM (that's difference I can see)..
from batch-insights.
jeroenwo
commented on May 20, 2024
Masking the instrumentation key might indeed be a good idea.. :D
from batch-insights.
timotheeguerin
commented on May 20, 2024
Yeah that probably means it didn't run. Could you send the stderr and stdout if possible
from batch-insights.
jeroenwo
commented on May 20, 2024
stderr.txt (I thought it was empty but apparently not)
You are using pip version 9.0.1, however version 9.0.3 is available.
You should consider upgrading via the 'python -m pip install --upgrade pip' command.
stdout.txt
Getting latest version of the Chocolatey package for download.
Getting Chocolatey from https://chocolatey.org/api/v2/package/chocolatey/0.10.9.
Downloading 7-Zip commandline tool prior to extraction.
Extracting D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall\chocolatey.zip to D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall...
Installing chocolatey on this machine
Creating ChocolateyInstall as an environment variable (targeting 'Machine')
Setting ChocolateyInstall to 'C:\ProgramData\chocolatey'
WARNING: It's very likely you will need to close and reopen your shell
before you can use choco.
Restricting write permissions to Administrators
We are setting up the Chocolatey package repository.
The packages themselves go to 'C:\ProgramData\chocolatey\lib'
(i.e. C:\ProgramData\chocolatey\lib\yourPackageName).
A shim file for the command line goes to 'C:\ProgramData\chocolatey\bin'
and points to an executable in 'C:\ProgramData\chocolatey\lib\yourPackageName'.
Creating Chocolatey folders if they do not already exist.
WARNING: You can safely ignore errors related to missing log files when
upgrading from a version of Chocolatey less than 0.9.9.
'Batch file could not be found' is also safe to ignore.
'The system cannot find the file specified' - also safe.
chocolatey.nupkg file not installed in lib.
Attempting to locate it from bootstrapper.
PATH environment variable does not have C:\ProgramData\chocolatey\bin in it. Adding...
WARNING: Not setting tab completion: Profile file does not exist at
'D:\Users\_azbatchtask_start\Documents\WindowsPowerShell\Microsoft.PowerShell_p
rofile.ps1'.
Chocolatey (choco.exe) is now ready.
You can call choco from anywhere, command line or powershell by typing choco.
Run choco /? for a list of functions.
You may need to shut down and restart powershell and/or consoles
first prior to using choco.
Ensuring chocolatey commands are on the path
Ensuring chocolatey.nupkg is in the lib folder
Chocolatey v0.10.9
Installing the following packages:
python
By installing you accept licenses for the packages.
Progress: Downloading python3 3.6.3... 13%
Progress: Downloading python3 3.6.3... 39%
Progress: Downloading python3 3.6.3... 66%
Progress: Downloading python3 3.6.3... 93%
Progress: Downloading python3 3.6.3... 100%
Progress: Downloading python 3.6.3... 20%
Progress: Downloading python 3.6.3... 63%
Progress: Downloading python 3.6.3... 100%
python3 v3.6.3 [Approved]
python3 package files install completed. Performing other installation steps.
Downloading python3 64 bit
from 'https://www.python.org/ftp/python/3.6.3/python-3.6.3-amd64.exe'
Progress: 0% - Saving 26.78 KB of 30.16 MB
Progress: 0% - Saving 160 KB of 30.16 MB
Progress: 1% - Saving 320 KB of 30.16 MB
Progress: 1% - Saving 480 KB of 30.16 MB
Progress: 2% - Saving 640 KB of 30.16 MB
Progress: 2% - Saving 800 KB of 30.16 MB
Progress: 3% - Saving 960 KB of 30.16 MB
Progress: 3% - Saving 1.09 MB of 30.16 MB
Progress: 4% - Saving 1.25 MB of 30.16 MB
Progress: 4% - Saving 1.41 MB of 30.16 MB
Progress: 5% - Saving 1.56 MB of 30.16 MB
Progress: 5% - Saving 1.72 MB of 30.16 MB
Progress: 6% - Saving 1.88 MB of 30.16 MB
Progress: 6% - Saving 2.03 MB of 30.16 MB
Progress: 7% - Saving 2.19 MB of 30.16 MB
Progress: 7% - Saving 2.34 MB of 30.16 MB
Progress: 8% - Saving 2.5 MB of 30.16 MB
Progress: 8% - Saving 2.66 MB of 30.16 MB
Progress: 9% - Saving 2.81 MB of 30.16 MB
Progress: 9% - Saving 2.97 MB of 30.16 MB
Progress: 10% - Saving 3.13 MB of 30.16 MB
Progress: 10% - Saving 3.28 MB of 30.16 MB
Progress: 11% - Saving 3.44 MB of 30.16 MB
Progress: 11% - Saving 3.59 MB of 30.16 MB
Progress: 12% - Saving 3.75 MB of 30.16 MB
Progress: 12% - Saving 3.91 MB of 30.16 MB
Progress: 13% - Saving 4.06 MB of 30.16 MB
Progress: 13% - Saving 4.22 MB of 30.16 MB
Progress: 14% - Saving 4.38 MB of 30.16 MB
Progress: 15% - Saving 4.53 MB of 30.16 MB
Progress: 15% - Saving 4.69 MB of 30.16 MB
Progress: 16% - Saving 4.84 MB of 30.16 MB
Progress: 16% - Saving 5 MB of 30.16 MB
Progress: 17% - Saving 5.16 MB of 30.16 MB
Progress: 17% - Saving 5.31 MB of 30.16 MB
Progress: 18% - Saving 5.47 MB of 30.16 MB
Progress: 18% - Saving 5.63 MB of 30.16 MB
Progress: 19% - Saving 5.78 MB of 30.16 MB
Progress: 19% - Saving 5.94 MB of 30.16 MB
Progress: 20% - Saving 6.09 MB of 30.16 MB
Progress: 20% - Saving 6.25 MB of 30.16 MB
Progress: 21% - Saving 6.41 MB of 30.16 MB
Progress: 21% - Saving 6.56 MB of 30.16 MB
Progress: 22% - Saving 6.72 MB of 30.16 MB
Progress: 22% - Saving 6.88 MB of 30.16 MB
Progress: 23% - Saving 7.03 MB of 30.16 MB
Progress: 23% - Saving 7.19 MB of 30.16 MB
Progress: 24% - Saving 7.34 MB of 30.16 MB
Progress: 24% - Saving 7.5 MB of 30.16 MB
Progress: 25% - Saving 7.66 MB of 30.16 MB
Progress: 25% - Saving 7.81 MB of 30.16 MB
Progress: 26% - Saving 7.97 MB of 30.16 MB
Progress: 26% - Saving 8.13 MB of 30.16 MB
Progress: 27% - Saving 8.28 MB of 30.16 MB
Progress: 27% - Saving 8.44 MB of 30.16 MB
Progress: 28% - Saving 8.59 MB of 30.16 MB
Progress: 29% - Saving 8.75 MB of 30.16 MB
Progress: 29% - Saving 8.91 MB of 30.16 MB
Progress: 30% - Saving 9.06 MB of 30.16 MB
Progress: 30% - Saving 9.22 MB of 30.16 MB
Progress: 31% - Saving 9.38 MB of 30.16 MB
Progress: 31% - Saving 9.53 MB of 30.16 MB
Progress: 32% - Saving 9.69 MB of 30.16 MB
Progress: 32% - Saving 9.84 MB of 30.16 MB
Progress: 33% - Saving 10 MB of 30.16 MB
Progress: 33% - Saving 10.16 MB of 30.16 MB
Progress: 34% - Saving 10.31 MB of 30.16 MB
Progress: 34% - Saving 10.47 MB of 30.16 MB
Progress: 35% - Saving 10.63 MB of 30.16 MB
Progress: 35% - Saving 10.78 MB of 30.16 MB
Progress: 36% - Saving 10.94 MB of 30.16 MB
Progress: 36% - Saving 11.09 MB of 30.16 MB
Progress: 37% - Saving 11.25 MB of 30.16 MB
Progress: 37% - Saving 11.41 MB of 30.16 MB
Progress: 38% - Saving 11.56 MB of 30.16 MB
Progress: 38% - Saving 11.72 MB of 30.16 MB
Progress: 39% - Saving 11.88 MB of 30.16 MB
Progress: 39% - Saving 12.03 MB of 30.16 MB
Progress: 40% - Saving 12.19 MB of 30.16 MB
Progress: 40% - Saving 12.34 MB of 30.16 MB
Progress: 41% - Saving 12.5 MB of 30.16 MB
Progress: 41% - Saving 12.66 MB of 30.16 MB
Progress: 42% - Saving 12.81 MB of 30.16 MB
Progress: 43% - Saving 12.97 MB of 30.16 MB
Progress: 43% - Saving 13.13 MB of 30.16 MB
Progress: 44% - Saving 13.28 MB of 30.16 MB
Progress: 44% - Saving 13.44 MB of 30.16 MB
Progress: 45% - Saving 13.59 MB of 30.16 MB
Progress: 45% - Saving 13.75 MB of 30.16 MB
Progress: 46% - Saving 13.91 MB of 30.16 MB
Progress: 46% - Saving 14.06 MB of 30.16 MB
Progress: 47% - Saving 14.22 MB of 30.16 MB
Progress: 47% - Saving 14.38 MB of 30.16 MB
Progress: 48% - Saving 14.53 MB of 30.16 MB
Progress: 48% - Saving 14.69 MB of 30.16 MB
Progress: 49% - Saving 14.84 MB of 30.16 MB
Progress: 49% - Saving 15 MB of 30.16 MB
Progress: 50% - Saving 15.16 MB of 30.16 MB
Progress: 50% - Saving 15.31 MB of 30.16 MB
Progress: 51% - Saving 15.47 MB of 30.16 MB
Progress: 51% - Saving 15.63 MB of 30.16 MB
Progress: 52% - Saving 15.78 MB of 30.16 MB
Progress: 52% - Saving 15.94 MB of 30.16 MB
Progress: 53% - Saving 16.09 MB of 30.16 MB
Progress: 53% - Saving 16.25 MB of 30.16 MB
Progress: 54% - Saving 16.41 MB of 30.16 MB
Progress: 54% - Saving 16.56 MB of 30.16 MB
Progress: 55% - Saving 16.72 MB of 30.16 MB
Progress: 55% - Saving 16.88 MB of 30.16 MB
Progress: 56% - Saving 17.03 MB of 30.16 MB
Progress: 56% - Saving 17.19 MB of 30.16 MB
Progress: 57% - Saving 17.34 MB of 30.16 MB
Progress: 58% - Saving 17.5 MB of 30.16 MB
Progress: 58% - Saving 17.66 MB of 30.16 MB
Progress: 59% - Saving 17.81 MB of 30.16 MB
Progress: 59% - Saving 17.97 MB of 30.16 MB
Progress: 60% - Saving 18.13 MB of 30.16 MB
Progress: 60% - Saving 18.28 MB of 30.16 MB
Progress: 61% - Saving 18.44 MB of 30.16 MB
Progress: 61% - Saving 18.59 MB of 30.16 MB
Progress: 62% - Saving 18.75 MB of 30.16 MB
Progress: 62% - Saving 18.91 MB of 30.16 MB
Progress: 63% - Saving 19.06 MB of 30.16 MB
Progress: 63% - Saving 19.22 MB of 30.16 MB
Progress: 64% - Saving 19.38 MB of 30.16 MB
Progress: 64% - Saving 19.53 MB of 30.16 MB
Progress: 65% - Saving 19.69 MB of 30.16 MB
Progress: 65% - Saving 19.84 MB of 30.16 MB
Progress: 66% - Saving 20 MB of 30.16 MB
Progress: 66% - Saving 20.16 MB of 30.16 MB
Progress: 67% - Saving 20.31 MB of 30.16 MB
Progress: 67% - Saving 20.47 MB of 30.16 MB
Progress: 68% - Saving 20.63 MB of 30.16 MB
Progress: 68% - Saving 20.78 MB of 30.16 MB
Progress: 69% - Saving 20.94 MB of 30.16 MB
Progress: 69% - Saving 21.09 MB of 30.16 MB
Progress: 70% - Saving 21.25 MB of 30.16 MB
Progress: 70% - Saving 21.41 MB of 30.16 MB
Progress: 71% - Saving 21.56 MB of 30.16 MB
Progress: 72% - Saving 21.72 MB of 30.16 MB
Progress: 72% - Saving 21.88 MB of 30.16 MB
Progress: 73% - Saving 22.03 MB of 30.16 MB
Progress: 73% - Saving 22.19 MB of 30.16 MB
Progress: 74% - Saving 22.34 MB of 30.16 MB
Progress: 74% - Saving 22.5 MB of 30.16 MB
Progress: 75% - Saving 22.66 MB of 30.16 MB
Progress: 75% - Saving 22.81 MB of 30.16 MB
Progress: 76% - Saving 22.97 MB of 30.16 MB
Progress: 76% - Saving 23.13 MB of 30.16 MB
Progress: 77% - Saving 23.28 MB of 30.16 MB
Progress: 77% - Saving 23.44 MB of 30.16 MB
Progress: 78% - Saving 23.59 MB of 30.16 MB
Progress: 78% - Saving 23.75 MB of 30.16 MB
Progress: 79% - Saving 23.91 MB of 30.16 MB
Progress: 79% - Saving 24.06 MB of 30.16 MB
Progress: 80% - Saving 24.22 MB of 30.16 MB
Progress: 80% - Saving 24.38 MB of 30.16 MB
Progress: 81% - Saving 24.53 MB of 30.16 MB
Progress: 81% - Saving 24.69 MB of 30.16 MB
Progress: 82% - Saving 24.84 MB of 30.16 MB
Progress: 82% - Saving 25 MB of 30.16 MB
Progress: 83% - Saving 25.16 MB of 30.16 MB
Progress: 83% - Saving 25.31 MB of 30.16 MB
Progress: 84% - Saving 25.47 MB of 30.16 MB
Progress: 84% - Saving 25.63 MB of 30.16 MB
Progress: 85% - Saving 25.78 MB of 30.16 MB
Progress: 86% - Saving 25.94 MB of 30.16 MB
Progress: 86% - Saving 26.09 MB of 30.16 MB
Progress: 87% - Saving 26.25 MB of 30.16 MB
Progress: 87% - Saving 26.41 MB of 30.16 MB
Progress: 88% - Saving 26.56 MB of 30.16 MB
Progress: 88% - Saving 26.72 MB of 30.16 MB
Progress: 89% - Saving 26.88 MB of 30.16 MB
Progress: 89% - Saving 27.03 MB of 30.16 MB
Progress: 90% - Saving 27.19 MB of 30.16 MB
Progress: 90% - Saving 27.34 MB of 30.16 MB
Progress: 91% - Saving 27.5 MB of 30.16 MB
Progress: 91% - Saving 27.66 MB of 30.16 MB
Progress: 92% - Saving 27.81 MB of 30.16 MB
Progress: 92% - Saving 27.97 MB of 30.16 MB
Progress: 93% - Saving 28.13 MB of 30.16 MB
Progress: 93% - Saving 28.28 MB of 30.16 MB
Progress: 94% - Saving 28.44 MB of 30.16 MB
Progress: 94% - Saving 28.59 MB of 30.16 MB
Progress: 95% - Saving 28.75 MB of 30.16 MB
Progress: 95% - Saving 28.91 MB of 30.16 MB
Progress: 96% - Saving 29.06 MB of 30.16 MB
Progress: 96% - Saving 29.21 MB of 30.16 MB
Progress: 97% - Saving 29.36 MB of 30.16 MB
Progress: 97% - Saving 29.52 MB of 30.16 MB
Progress: 98% - Saving 29.67 MB of 30.16 MB
Progress: 98% - Saving 29.83 MB of 30.16 MB
Progress: 99% - Saving 29.99 MB of 30.16 MB
Progress: 99% - Saving 30.14 MB of 30.16 MB
Progress: 100% - Completed download of D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\python3\3.6.3\python-3.6.3-amd64.exe (30.16 MB).
Download of python-3.6.3-amd64.exe (30.16 MB) completed.
Hashes match.
Installing python3...
python3 has been installed.
Installed to 'C:\Python36'
python3 can be automatically uninstalled.
Environment Vars (like PATH) have changed. Close/reopen your shell to
see the changes (or in powershell/cmd.exe just type `refreshenv`).
The install of python3 was successful.
Software installed as 'EXE', install location is likely default.
python v3.6.3 [Approved]
python package files install completed. Performing other installation steps.
The install of python was successful.
Software install location not explicitly set, could be in package or
default install location if installer.
Chocolatey installed 2/2 packages.
See the log for details (C:\ProgramData\chocolatey\logs\chocolatey.log).
Enjoy using Chocolatey? Explore more amazing features to take your
experience to the next level at
https://chocolatey.org/compare
Current path: C:\Python36\Scripts\;C:\Python36\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\ProgramData\chocolatey\bin;;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;D:\batch\tasks\shared;D:\batch\tasks\startup\wd
Python version:
Python 3.6.3
Collecting psutil
Downloading psutil-5.4.3-cp36-cp36m-win_amd64.whl (226kB)
Collecting python-dateutil
Downloading python_dateutil-2.7.1-py2.py3-none-any.whl (212kB)
Collecting applicationinsights
Downloading applicationinsights-0.11.2.tar.gz (45kB)
Collecting six>=1.5 (from python-dateutil)
Downloading six-1.11.0-py2.py3-none-any.whl
Installing collected packages: psutil, six, python-dateutil, applicationinsights
Running setup.py install for applicationinsights: started
Running setup.py install for applicationinsights: finished with status 'done'
Successfully installed applicationinsights-0.11.2 psutil-5.4.3 python-dateutil-2.7.1 six-1.11.0
Downloading nodestats.py
Starting App insights background process in D:\batch\tasks\startup\wd
TaskPath TaskName
-------- --------
\ batchappinsights
\ batchappinsights
from batch-insights.
timotheeguerin
commented on May 20, 2024
Ok I think I fixed it now, just double checking a few things now
from batch-insights.
jeroenwo
commented on May 20, 2024
node-stats.err.log
Traceback (most recent call last):
File ".\nodestats.py", line 14, in <module>
from applicationinsights import TelemetryClient
ImportError: No module named applicationinsights
node-stats.err.log
(empty)node-stats.err.log
"""TVM stats"""
# stdlib imports
import logging
from datetime import datetime
import os
import time
import platform
from collections import namedtuple
import sys
# non-stdlib imports
import psutil
from applicationinsights import TelemetryClient
VERSION = "0.0.1.1"
_DEFAULT_STATS_UPDATE_INTERVAL = 5
def setup_logger():
# logger defines
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s.%(msecs)03dZ %(levelname)s %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
logger = setup_logger()
# global defines
_IS_PLATFORM_WINDOWS = platform.system() == 'Windows'
def python_environment(): # pragma: no cover
"""
Returns the current python environment information
"""
return ' '.join(
[platform.python_implementation(),
platform.python_version()])
def os_environment():
"""
Get the OS environment
"""
return platform.platform()
def is_windows():
"""
:returns: If running on windows
"""
return _IS_PLATFORM_WINDOWS
def avg(list):
"""
Compute the average of a list
"""
return sum(list) / float(len(list))
def pretty_nb(num, suffix=''):
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1000.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1000.0
return "%.1f%s%s" % (num, 'Yi', suffix)
NodeIOStats = namedtuple('NodeIOStats', ['read_bps', 'write_bps'])
class NodeStats:
"""Persistent Task Stats class"""
def __init__(self,
num_connected_users=0,
num_pids=0,
cpu_count=0,
cpu_percent=None,
mem_total=0,
mem_avail=0,
swap_total=0,
swap_avail=0,
disk_total=0,
disk=None,
net=None):
"""
Map the attributes
"""
self.num_connected_users = num_connected_users
self.num_pids = num_pids
self.cpu_count = cpu_count
self.cpu_percent = cpu_percent
self.mem_total = mem_total
self.mem_avail = mem_avail
self.swap_total = swap_total
self.swap_avail = swap_avail
self.disk_total = disk_total
self.disk = disk or NodeIOStats()
self.net = net or NodeIOStats()
@property
def mem_used(self):
"""
Return the memory used
"""
return self.mem_total - self.mem_avail
class IOThroughputAggregator:
def __init__(self):
self.last_timestamp = None
self.last_read = 0
self.last_write = 0
def aggregate(self, cur_read, cur_write):
"""
Aggregate with the new values
"""
now = datetime.now()
read_bps = 0
write_bps = 0
if self.last_timestamp:
delta = (now - self.last_timestamp).total_seconds()
read_bps = (cur_read - self.last_read) / delta
write_bps = (cur_write - self.last_write) / delta
self.last_timestamp = now
self.last_read = cur_read
self.last_write = cur_write
return NodeIOStats(read_bps, write_bps)
class NodeStatsCollector:
"""
Node Stats Manager class
"""
def __init__(self,
pool_id,
node_id,
refresh_interval=_DEFAULT_STATS_UPDATE_INTERVAL,
app_insights_key=None):
self.pool_id = pool_id
self.node_id = node_id
self.telemetry_client = None
self.first_collect = True
self.refresh_interval = refresh_interval
self.disk = IOThroughputAggregator()
self.network = IOThroughputAggregator()
if app_insights_key or 'APP_INSIGHTS_INSTRUMENTATION_KEY' in os.environ or 'APP_INSIGHTS_KEY' in os.environ:
key = (app_insights_key
or os.environ.get('APP_INSIGHTS_INSTRUMENTATION_KEY')
or os.environ.get('APP_INSIGHTS_KEY'))
logger.info("Detected instrumentation key. Will upload stats to app insights")
self.telemetry_client = TelemetryClient(key)
context = self.telemetry_client.context
context.application.id = 'AzureBatchInsights'
context.application.ver = VERSION
context.device.model = "BatchNode"
context.device.role_name = self.pool_id
context.device.role_instance = self.node_id
else:
logger.info("No instrumentation key detected. Cannot upload to app insights." +
"Make sure you have the APP_INSIGHTS_INSTRUMENTATION_KEY environment variable setup")
def init(self):
"""
Initialize the monitoring
"""
# start cpu utilization monitoring, first value is ignored
psutil.cpu_percent(interval=None, percpu=True)
def _get_network_usage(self):
netio = psutil.net_io_counters()
return self.network.aggregate(netio.bytes_recv, netio.bytes_sent)
def _get_disk_usage(self):
diskio = psutil.disk_io_counters()
return self.disk.aggregate(diskio.read_bytes, diskio.write_bytes)
def _sample_stats(self):
# get system-wide counters
mem = psutil.virtual_memory()
disk_stats = self._get_disk_usage()
net_stats = self._get_network_usage()
swap_total, _, swap_avail, _, _, _ = psutil.swap_memory()
stats = NodeStats(
cpu_count=psutil.cpu_count(),
cpu_percent=psutil.cpu_percent(interval=None, percpu=True),
num_pids=len(psutil.pids()),
# Memory
mem_total=mem.total,
mem_avail=mem.available,
swap_total=swap_total,
swap_avail=swap_avail,
# Disk IO
disk=disk_stats,
# Net transfer
net=net_stats,
)
del mem
return stats
def _collect_stats(self):
"""
Collect the stats and then send to app insights
"""
# collect stats
stats = self._sample_stats()
if self.first_collect:
self.first_collect = False
return
if stats is None:
logger.error("Could not sample node stats")
return
if self.telemetry_client:
self._send_stats(stats)
else:
self._log_stats(stats)
def _send_stats(self, stats):
"""
Retrieve the current stats and send to app insights
"""
process = psutil.Process(os.getpid())
logger.debug("Uploading stats. Mem of this script: %d vs total: %d",
process.memory_info().rss, stats.mem_avail)
client = self.telemetry_client
for cpu_n in range(0, stats.cpu_count):
client.track_metric(
"Cpu usage",
stats.cpu_percent[cpu_n],
properties={
"Cpu #": cpu_n
})
client.track_metric("Memory used", stats.mem_used)
client.track_metric("Memory available", stats.mem_avail)
client.track_metric("Disk read", stats.disk.read_bps)
client.track_metric("Disk write", stats.disk.write_bps)
client.track_metric("Network read", stats.net.read_bps)
client.track_metric("Network write", stats.net.write_bps)
self.telemetry_client.flush()
def _log_stats(self, stats):
logger.info(
"========================= Stats =========================")
logger.info("Cpu percent: %d%% %s", avg(stats.cpu_percent),
stats.cpu_percent)
logger.info("Memory used: %sB / %sB", pretty_nb(stats.mem_used),
pretty_nb(stats.mem_total))
logger.info("Swap used: %sB / %sB", pretty_nb(
stats.swap_avail), pretty_nb(stats.swap_total))
logger.info("Net read: %sBs",
pretty_nb(stats.net.read_bps))
logger.info("Net write: %sBs",
pretty_nb(stats.net.write_bps))
logger.info("Disk read: %sBs",
pretty_nb(stats.disk.read_bps))
logger.info("Disk write: %sBs",
pretty_nb(stats.disk.write_bps))
logger.info("-------------------------------------")
logger.info("")
def run(self):
"""
Start collecting information of the system.
"""
logger.debug("Start collecting stats for pool=%s node=%s",
self.pool_id, self.node_id)
while True:
self._collect_stats()
time.sleep(self.refresh_interval)
def main():
"""
Main entry point for prism
"""
# log basic info
logger.info("Python args: %s", sys.argv)
logger.info("Python interpreter: %s", python_environment())
logger.info("Operating system: %s", os_environment())
logger.info("Cpu count: %s", psutil.cpu_count())
pool_id = os.environ.get('AZ_BATCH_POOL_ID', '_test-pool-1')
node_id = os.environ.get('AZ_BATCH_NODE_ID', '_test-node-1')
# get and set event loop mode
logger.info('enabling event loop debug mode')
app_insights_key = None
if len(sys.argv) > 2:
pool_id = sys.argv[1]
node_id = sys.argv[2]
if len(sys.argv) > 3:
app_insights_key = sys.argv[3]
# create node stats manager
collector = NodeStatsCollector(
pool_id, node_id, app_insights_key=app_insights_key)
collector.init()
collector.run()
if __name__ == '__main__':
main()stderr.txt
You are using pip version 9.0.1, however version 9.0.3 is available.
You should consider upgrading via the 'python -m pip install --upgrade pip' command.
Get-ScheduledTask : No MSFT_ScheduledTask objects found with property
'TaskName' equal to 'batchappinsights'. Verify the value of the property and
retry.
At line:13 char:11
+ $exists = Get-ScheduledTask -TaskName "batchappinsights";
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : ObjectNotFound: (batchappinsights:String) [Get-S
cheduledTask], CimJobException
+ FullyQualifiedErrorId : CmdletizationQuery_NotFound_TaskName,Get-Schedul
edTask
stdout.txt
Getting latest version of the Chocolatey package for download.
Getting Chocolatey from https://chocolatey.org/api/v2/package/chocolatey/0.10.9.
Downloading 7-Zip commandline tool prior to extraction.
Extracting D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall\chocolatey.zip to D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall...
Installing chocolatey on this machine
Creating ChocolateyInstall as an environment variable (targeting 'Machine')
Setting ChocolateyInstall to 'C:\ProgramData\chocolatey'
WARNING: It's very likely you will need to close and reopen your shell
before you can use choco.
Restricting write permissions to Administrators
We are setting up the Chocolatey package repository.
The packages themselves go to 'C:\ProgramData\chocolatey\lib'
(i.e. C:\ProgramData\chocolatey\lib\yourPackageName).
A shim file for the command line goes to 'C:\ProgramData\chocolatey\bin'
and points to an executable in 'C:\ProgramData\chocolatey\lib\yourPackageName'.
Creating Chocolatey folders if they do not already exist.
WARNING: You can safely ignore errors related to missing log files when
upgrading from a version of Chocolatey less than 0.9.9.
'Batch file could not be found' is also safe to ignore.
'The system cannot find the file specified' - also safe.
chocolatey.nupkg file not installed in lib.
Attempting to locate it from bootstrapper.
PATH environment variable does not have C:\ProgramData\chocolatey\bin in it. Adding...
WARNING: Not setting tab completion: Profile file does not exist at
'D:\Users\_azbatchtask_start\Documents\WindowsPowerShell\Microsoft.PowerShell_p
rofile.ps1'.
Chocolatey (choco.exe) is now ready.
You can call choco from anywhere, command line or powershell by typing choco.
Run choco /? for a list of functions.
You may need to shut down and restart powershell and/or consoles
first prior to using choco.
Ensuring chocolatey commands are on the path
Ensuring chocolatey.nupkg is in the lib folder
Chocolatey v0.10.9
Installing the following packages:
python
By installing you accept licenses for the packages.
Progress: Downloading python3 3.6.3... 12%
Progress: Downloading python3 3.6.3... 39%
Progress: Downloading python3 3.6.3... 66%
Progress: Downloading python3 3.6.3... 92%
Progress: Downloading python3 3.6.3... 100%
Progress: Downloading python 3.6.3... 20%
Progress: Downloading python 3.6.3... 62%
Progress: Downloading python 3.6.3... 100%
python3 v3.6.3 [Approved]
python3 package files install completed. Performing other installation steps.
Downloading python3 64 bit
from 'https://www.python.org/ftp/python/3.6.3/python-3.6.3-amd64.exe'
Progress: 0% - Saving 26.27 KB of 30.16 MB
Progress: 0% - Saving 159.49 KB of 30.16 MB
Progress: 1% - Saving 319.49 KB of 30.16 MB
Progress: 1% - Saving 479.49 KB of 30.16 MB
Progress: 2% - Saving 639.49 KB of 30.16 MB
Progress: 2% - Saving 799.49 KB of 30.16 MB
Progress: 3% - Saving 959.49 KB of 30.16 MB
Progress: 3% - Saving 1.09 MB of 30.16 MB
Progress: 4% - Saving 1.25 MB of 30.16 MB
Progress: 4% - Saving 1.41 MB of 30.16 MB
Progress: 5% - Saving 1.56 MB of 30.16 MB
Progress: 5% - Saving 1.72 MB of 30.16 MB
Progress: 6% - Saving 1.87 MB of 30.16 MB
Progress: 6% - Saving 2.03 MB of 30.16 MB
Progress: 7% - Saving 2.19 MB of 30.16 MB
Progress: 7% - Saving 2.34 MB of 30.16 MB
Progress: 8% - Saving 2.5 MB of 30.16 MB
Progress: 8% - Saving 2.66 MB of 30.16 MB
Progress: 9% - Saving 2.81 MB of 30.16 MB
Progress: 9% - Saving 2.97 MB of 30.16 MB
Progress: 10% - Saving 3.12 MB of 30.16 MB
Progress: 10% - Saving 3.28 MB of 30.16 MB
Progress: 11% - Saving 3.44 MB of 30.16 MB
Progress: 11% - Saving 3.59 MB of 30.16 MB
Progress: 12% - Saving 3.75 MB of 30.16 MB
Progress: 12% - Saving 3.91 MB of 30.16 MB
Progress: 13% - Saving 4.06 MB of 30.16 MB
Progress: 13% - Saving 4.22 MB of 30.16 MB
Progress: 14% - Saving 4.37 MB of 30.16 MB
Progress: 15% - Saving 4.53 MB of 30.16 MB
Progress: 15% - Saving 4.69 MB of 30.16 MB
Progress: 16% - Saving 4.84 MB of 30.16 MB
Progress: 16% - Saving 5 MB of 30.16 MB
Progress: 17% - Saving 5.16 MB of 30.16 MB
Progress: 17% - Saving 5.31 MB of 30.16 MB
Progress: 18% - Saving 5.47 MB of 30.16 MB
Progress: 18% - Saving 5.62 MB of 30.16 MB
Progress: 19% - Saving 5.78 MB of 30.16 MB
Progress: 19% - Saving 5.94 MB of 30.16 MB
Progress: 20% - Saving 6.09 MB of 30.16 MB
Progress: 20% - Saving 6.25 MB of 30.16 MB
Progress: 21% - Saving 6.41 MB of 30.16 MB
Progress: 21% - Saving 6.56 MB of 30.16 MB
Progress: 22% - Saving 6.72 MB of 30.16 MB
Progress: 22% - Saving 6.87 MB of 30.16 MB
Progress: 23% - Saving 7.03 MB of 30.16 MB
Progress: 23% - Saving 7.19 MB of 30.16 MB
Progress: 24% - Saving 7.34 MB of 30.16 MB
Progress: 24% - Saving 7.5 MB of 30.16 MB
Progress: 25% - Saving 7.66 MB of 30.16 MB
Progress: 25% - Saving 7.81 MB of 30.16 MB
Progress: 26% - Saving 7.97 MB of 30.16 MB
Progress: 26% - Saving 8.12 MB of 30.16 MB
Progress: 27% - Saving 8.28 MB of 30.16 MB
Progress: 27% - Saving 8.44 MB of 30.16 MB
Progress: 28% - Saving 8.59 MB of 30.16 MB
Progress: 29% - Saving 8.75 MB of 30.16 MB
Progress: 29% - Saving 8.91 MB of 30.16 MB
Progress: 30% - Saving 9.06 MB of 30.16 MB
Progress: 30% - Saving 9.22 MB of 30.16 MB
Progress: 31% - Saving 9.37 MB of 30.16 MB
Progress: 31% - Saving 9.53 MB of 30.16 MB
Progress: 32% - Saving 9.69 MB of 30.16 MB
Progress: 32% - Saving 9.84 MB of 30.16 MB
Progress: 33% - Saving 10 MB of 30.16 MB
Progress: 33% - Saving 10.16 MB of 30.16 MB
Progress: 34% - Saving 10.31 MB of 30.16 MB
Progress: 34% - Saving 10.47 MB of 30.16 MB
Progress: 35% - Saving 10.62 MB of 30.16 MB
Progress: 35% - Saving 10.78 MB of 30.16 MB
Progress: 36% - Saving 10.94 MB of 30.16 MB
Progress: 36% - Saving 11.09 MB of 30.16 MB
Progress: 37% - Saving 11.25 MB of 30.16 MB
Progress: 37% - Saving 11.41 MB of 30.16 MB
Progress: 38% - Saving 11.56 MB of 30.16 MB
Progress: 38% - Saving 11.72 MB of 30.16 MB
Progress: 39% - Saving 11.87 MB of 30.16 MB
Progress: 39% - Saving 12.03 MB of 30.16 MB
Progress: 40% - Saving 12.19 MB of 30.16 MB
Progress: 40% - Saving 12.34 MB of 30.16 MB
Progress: 41% - Saving 12.5 MB of 30.16 MB
Progress: 41% - Saving 12.66 MB of 30.16 MB
Progress: 42% - Saving 12.81 MB of 30.16 MB
Progress: 43% - Saving 12.97 MB of 30.16 MB
Progress: 43% - Saving 13.12 MB of 30.16 MB
Progress: 44% - Saving 13.28 MB of 30.16 MB
Progress: 44% - Saving 13.44 MB of 30.16 MB
Progress: 45% - Saving 13.59 MB of 30.16 MB
Progress: 45% - Saving 13.75 MB of 30.16 MB
Progress: 46% - Saving 13.91 MB of 30.16 MB
Progress: 46% - Saving 14.06 MB of 30.16 MB
Progress: 47% - Saving 14.22 MB of 30.16 MB
Progress: 47% - Saving 14.37 MB of 30.16 MB
Progress: 48% - Saving 14.53 MB of 30.16 MB
Progress: 48% - Saving 14.69 MB of 30.16 MB
Progress: 49% - Saving 14.84 MB of 30.16 MB
Progress: 49% - Saving 15 MB of 30.16 MB
Progress: 50% - Saving 15.16 MB of 30.16 MB
Progress: 50% - Saving 15.31 MB of 30.16 MB
Progress: 51% - Saving 15.47 MB of 30.16 MB
Progress: 51% - Saving 15.62 MB of 30.16 MB
Progress: 52% - Saving 15.78 MB of 30.16 MB
Progress: 52% - Saving 15.94 MB of 30.16 MB
Progress: 53% - Saving 16.09 MB of 30.16 MB
Progress: 53% - Saving 16.25 MB of 30.16 MB
Progress: 54% - Saving 16.41 MB of 30.16 MB
Progress: 54% - Saving 16.56 MB of 30.16 MB
Progress: 55% - Saving 16.72 MB of 30.16 MB
Progress: 55% - Saving 16.87 MB of 30.16 MB
Progress: 56% - Saving 17.03 MB of 30.16 MB
Progress: 56% - Saving 17.19 MB of 30.16 MB
Progress: 57% - Saving 17.34 MB of 30.16 MB
Progress: 58% - Saving 17.5 MB of 30.16 MB
Progress: 58% - Saving 17.66 MB of 30.16 MB
Progress: 59% - Saving 17.81 MB of 30.16 MB
Progress: 59% - Saving 17.97 MB of 30.16 MB
Progress: 60% - Saving 18.12 MB of 30.16 MB
Progress: 60% - Saving 18.28 MB of 30.16 MB
Progress: 61% - Saving 18.44 MB of 30.16 MB
Progress: 61% - Saving 18.59 MB of 30.16 MB
Progress: 62% - Saving 18.75 MB of 30.16 MB
Progress: 62% - Saving 18.91 MB of 30.16 MB
Progress: 63% - Saving 19.06 MB of 30.16 MB
Progress: 63% - Saving 19.22 MB of 30.16 MB
Progress: 64% - Saving 19.37 MB of 30.16 MB
Progress: 64% - Saving 19.53 MB of 30.16 MB
Progress: 65% - Saving 19.69 MB of 30.16 MB
Progress: 65% - Saving 19.84 MB of 30.16 MB
Progress: 66% - Saving 20 MB of 30.16 MB
Progress: 66% - Saving 20.16 MB of 30.16 MB
Progress: 67% - Saving 20.31 MB of 30.16 MB
Progress: 67% - Saving 20.47 MB of 30.16 MB
Progress: 68% - Saving 20.62 MB of 30.16 MB
Progress: 68% - Saving 20.78 MB of 30.16 MB
Progress: 69% - Saving 20.94 MB of 30.16 MB
Progress: 69% - Saving 21.09 MB of 30.16 MB
Progress: 70% - Saving 21.25 MB of 30.16 MB
Progress: 70% - Saving 21.41 MB of 30.16 MB
Progress: 71% - Saving 21.56 MB of 30.16 MB
Progress: 72% - Saving 21.72 MB of 30.16 MB
Progress: 72% - Saving 21.87 MB of 30.16 MB
Progress: 73% - Saving 22.03 MB of 30.16 MB
Progress: 73% - Saving 22.19 MB of 30.16 MB
Progress: 74% - Saving 22.34 MB of 30.16 MB
Progress: 74% - Saving 22.5 MB of 30.16 MB
Progress: 75% - Saving 22.66 MB of 30.16 MB
Progress: 75% - Saving 22.81 MB of 30.16 MB
Progress: 76% - Saving 22.97 MB of 30.16 MB
Progress: 76% - Saving 23.12 MB of 30.16 MB
Progress: 77% - Saving 23.28 MB of 30.16 MB
Progress: 77% - Saving 23.44 MB of 30.16 MB
Progress: 78% - Saving 23.59 MB of 30.16 MB
Progress: 78% - Saving 23.75 MB of 30.16 MB
Progress: 79% - Saving 23.91 MB of 30.16 MB
Progress: 79% - Saving 24.06 MB of 30.16 MB
Progress: 80% - Saving 24.22 MB of 30.16 MB
Progress: 80% - Saving 24.37 MB of 30.16 MB
Progress: 81% - Saving 24.53 MB of 30.16 MB
Progress: 81% - Saving 24.69 MB of 30.16 MB
Progress: 82% - Saving 24.84 MB of 30.16 MB
Progress: 82% - Saving 25 MB of 30.16 MB
Progress: 83% - Saving 25.16 MB of 30.16 MB
Progress: 83% - Saving 25.31 MB of 30.16 MB
Progress: 84% - Saving 25.47 MB of 30.16 MB
Progress: 84% - Saving 25.62 MB of 30.16 MB
Progress: 85% - Saving 25.78 MB of 30.16 MB
Progress: 86% - Saving 25.94 MB of 30.16 MB
Progress: 86% - Saving 26.09 MB of 30.16 MB
Progress: 87% - Saving 26.25 MB of 30.16 MB
Progress: 87% - Saving 26.41 MB of 30.16 MB
Progress: 88% - Saving 26.56 MB of 30.16 MB
Progress: 88% - Saving 26.72 MB of 30.16 MB
Progress: 89% - Saving 26.87 MB of 30.16 MB
Progress: 89% - Saving 27.03 MB of 30.16 MB
Progress: 90% - Saving 27.19 MB of 30.16 MB
Progress: 90% - Saving 27.34 MB of 30.16 MB
Progress: 91% - Saving 27.5 MB of 30.16 MB
Progress: 91% - Saving 27.66 MB of 30.16 MB
Progress: 92% - Saving 27.81 MB of 30.16 MB
Progress: 92% - Saving 27.97 MB of 30.16 MB
Progress: 93% - Saving 28.12 MB of 30.16 MB
Progress: 93% - Saving 28.28 MB of 30.16 MB
Progress: 94% - Saving 28.44 MB of 30.16 MB
Progress: 94% - Saving 28.59 MB of 30.16 MB
Progress: 95% - Saving 28.75 MB of 30.16 MB
Progress: 95% - Saving 28.91 MB of 30.16 MB
Progress: 96% - Saving 29.06 MB of 30.16 MB
Progress: 96% - Saving 29.22 MB of 30.16 MB
Progress: 97% - Saving 29.37 MB of 30.16 MB
Progress: 97% - Saving 29.53 MB of 30.16 MB
Progress: 98% - Saving 29.69 MB of 30.16 MB
Progress: 98% - Saving 29.84 MB of 30.16 MB
Progress: 99% - Saving 30 MB of 30.16 MB
Progress: 100% - Saving 30.16 MB of 30.16 MB
Progress: 100% - Completed download of D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\python3\3.6.3\python-3.6.3-amd64.exe (30.16 MB).
Download of python-3.6.3-amd64.exe (30.16 MB) completed.
Hashes match.
Installing python3...
python3 has been installed.
Installed to 'C:\Python36'
python3 can be automatically uninstalled.
Environment Vars (like PATH) have changed. Close/reopen your shell to
see the changes (or in powershell/cmd.exe just type `refreshenv`).
The install of python3 was successful.
Software installed as 'EXE', install location is likely default.
python v3.6.3 [Approved]
python package files install completed. Performing other installation steps.
The install of python was successful.
Software install location not explicitly set, could be in package or
default install location if installer.
Chocolatey installed 2/2 packages.
See the log for details (C:\ProgramData\chocolatey\logs\chocolatey.log).
Current path: C:\Python36\Scripts\;C:\Python36\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\ProgramData\chocolatey\bin;;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;D:\batch\tasks\shared;D:\batch\tasks\startup\wd
Python version:
Python 3.6.3
Collecting psutil
Downloading psutil-5.4.3-cp36-cp36m-win_amd64.whl (226kB)
Collecting python-dateutil
Downloading python_dateutil-2.7.2-py2.py3-none-any.whl (212kB)
Collecting applicationinsights
Downloading applicationinsights-0.11.2.tar.gz (45kB)
Collecting six>=1.5 (from python-dateutil)
Downloading six-1.11.0-py2.py3-none-any.whl
Installing collected packages: psutil, six, python-dateutil, applicationinsights
Running setup.py install for applicationinsights: started
Running setup.py install for applicationinsights: finished with status 'done'
Successfully installed applicationinsights-0.11.2 psutil-5.4.3 python-dateutil-2.7.2 six-1.11.0
Downloading nodestats.py
Starting App insights background process in D:\batch\tasks\startup\wd
TaskPath TaskName
-------- --------
\ batchappinsights
\ batchappinsights
from batch-insights.
timotheeguerin
commented on May 20, 2024
... Sorry about this, keep getting new issues
from batch-insights.
timotheeguerin
commented on May 20, 2024
Could you try once more. It should work hopefully this time
from batch-insights.
jeroenwo
commented on May 20, 2024
node-stats.err.log
Traceback (most recent call last):
File ".\nodestats.py", line 14, in <module>
from applicationinsights import TelemetryClient
ImportError: No module named applicationinsights
node-stats.log
(empty)nodestats.py
"""TVM stats"""
# stdlib imports
import logging
from datetime import datetime
import os
import time
import platform
from collections import namedtuple
import sys
# non-stdlib imports
import psutil
from applicationinsights import TelemetryClient
VERSION = "0.0.1.1"
_DEFAULT_STATS_UPDATE_INTERVAL = 5
def setup_logger():
# logger defines
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s.%(msecs)03dZ %(levelname)s %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
logger = setup_logger()
# global defines
_IS_PLATFORM_WINDOWS = platform.system() == 'Windows'
def python_environment(): # pragma: no cover
"""
Returns the current python environment information
"""
return ' '.join(
[platform.python_implementation(),
platform.python_version()])
def os_environment():
"""
Get the OS environment
"""
return platform.platform()
def is_windows():
"""
:returns: If running on windows
"""
return _IS_PLATFORM_WINDOWS
def avg(list):
"""
Compute the average of a list
"""
return sum(list) / float(len(list))
def pretty_nb(num, suffix=''):
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1000.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1000.0
return "%.1f%s%s" % (num, 'Yi', suffix)
NodeIOStats = namedtuple('NodeIOStats', ['read_bps', 'write_bps'])
class NodeStats:
"""Persistent Task Stats class"""
def __init__(self,
num_connected_users=0,
num_pids=0,
cpu_count=0,
cpu_percent=None,
mem_total=0,
mem_avail=0,
swap_total=0,
swap_avail=0,
disk_total=0,
disk=None,
net=None):
"""
Map the attributes
"""
self.num_connected_users = num_connected_users
self.num_pids = num_pids
self.cpu_count = cpu_count
self.cpu_percent = cpu_percent
self.mem_total = mem_total
self.mem_avail = mem_avail
self.swap_total = swap_total
self.swap_avail = swap_avail
self.disk_total = disk_total
self.disk = disk or NodeIOStats()
self.net = net or NodeIOStats()
@property
def mem_used(self):
"""
Return the memory used
"""
return self.mem_total - self.mem_avail
class IOThroughputAggregator:
def __init__(self):
self.last_timestamp = None
self.last_read = 0
self.last_write = 0
def aggregate(self, cur_read, cur_write):
"""
Aggregate with the new values
"""
now = datetime.now()
read_bps = 0
write_bps = 0
if self.last_timestamp:
delta = (now - self.last_timestamp).total_seconds()
read_bps = (cur_read - self.last_read) / delta
write_bps = (cur_write - self.last_write) / delta
self.last_timestamp = now
self.last_read = cur_read
self.last_write = cur_write
return NodeIOStats(read_bps, write_bps)
class NodeStatsCollector:
"""
Node Stats Manager class
"""
def __init__(self,
pool_id,
node_id,
refresh_interval=_DEFAULT_STATS_UPDATE_INTERVAL,
app_insights_key=None):
self.pool_id = pool_id
self.node_id = node_id
self.telemetry_client = None
self.first_collect = True
self.refresh_interval = refresh_interval
self.disk = IOThroughputAggregator()
self.network = IOThroughputAggregator()
if app_insights_key or 'APP_INSIGHTS_INSTRUMENTATION_KEY' in os.environ or 'APP_INSIGHTS_KEY' in os.environ:
key = (app_insights_key
or os.environ.get('APP_INSIGHTS_INSTRUMENTATION_KEY')
or os.environ.get('APP_INSIGHTS_KEY'))
logger.info("Detected instrumentation key. Will upload stats to app insights")
self.telemetry_client = TelemetryClient(key)
context = self.telemetry_client.context
context.application.id = 'AzureBatchInsights'
context.application.ver = VERSION
context.device.model = "BatchNode"
context.device.role_name = self.pool_id
context.device.role_instance = self.node_id
else:
logger.info("No instrumentation key detected. Cannot upload to app insights." +
"Make sure you have the APP_INSIGHTS_INSTRUMENTATION_KEY environment variable setup")
def init(self):
"""
Initialize the monitoring
"""
# start cpu utilization monitoring, first value is ignored
psutil.cpu_percent(interval=None, percpu=True)
def _get_network_usage(self):
netio = psutil.net_io_counters()
return self.network.aggregate(netio.bytes_recv, netio.bytes_sent)
def _get_disk_usage(self):
diskio = psutil.disk_io_counters()
return self.disk.aggregate(diskio.read_bytes, diskio.write_bytes)
def _sample_stats(self):
# get system-wide counters
mem = psutil.virtual_memory()
disk_stats = self._get_disk_usage()
net_stats = self._get_network_usage()
swap_total, _, swap_avail, _, _, _ = psutil.swap_memory()
stats = NodeStats(
cpu_count=psutil.cpu_count(),
cpu_percent=psutil.cpu_percent(interval=None, percpu=True),
num_pids=len(psutil.pids()),
# Memory
mem_total=mem.total,
mem_avail=mem.available,
swap_total=swap_total,
swap_avail=swap_avail,
# Disk IO
disk=disk_stats,
# Net transfer
net=net_stats,
)
del mem
return stats
def _collect_stats(self):
"""
Collect the stats and then send to app insights
"""
# collect stats
stats = self._sample_stats()
if self.first_collect:
self.first_collect = False
return
if stats is None:
logger.error("Could not sample node stats")
return
if self.telemetry_client:
self._send_stats(stats)
else:
self._log_stats(stats)
def _send_stats(self, stats):
"""
Retrieve the current stats and send to app insights
"""
process = psutil.Process(os.getpid())
logger.debug("Uploading stats. Mem of this script: %d vs total: %d",
process.memory_info().rss, stats.mem_avail)
client = self.telemetry_client
for cpu_n in range(0, stats.cpu_count):
client.track_metric(
"Cpu usage",
stats.cpu_percent[cpu_n],
properties={
"Cpu #": cpu_n
})
client.track_metric("Memory used", stats.mem_used)
client.track_metric("Memory available", stats.mem_avail)
client.track_metric("Disk read", stats.disk.read_bps)
client.track_metric("Disk write", stats.disk.write_bps)
client.track_metric("Network read", stats.net.read_bps)
client.track_metric("Network write", stats.net.write_bps)
self.telemetry_client.flush()
def _log_stats(self, stats):
logger.info(
"========================= Stats =========================")
logger.info("Cpu percent: %d%% %s", avg(stats.cpu_percent),
stats.cpu_percent)
logger.info("Memory used: %sB / %sB", pretty_nb(stats.mem_used),
pretty_nb(stats.mem_total))
logger.info("Swap used: %sB / %sB", pretty_nb(
stats.swap_avail), pretty_nb(stats.swap_total))
logger.info("Net read: %sBs",
pretty_nb(stats.net.read_bps))
logger.info("Net write: %sBs",
pretty_nb(stats.net.write_bps))
logger.info("Disk read: %sBs",
pretty_nb(stats.disk.read_bps))
logger.info("Disk write: %sBs",
pretty_nb(stats.disk.write_bps))
logger.info("-------------------------------------")
logger.info("")
def run(self):
"""
Start collecting information of the system.
"""
logger.debug("Start collecting stats for pool=%s node=%s",
self.pool_id, self.node_id)
while True:
self._collect_stats()
time.sleep(self.refresh_interval)
def main():
"""
Main entry point for prism
"""
# log basic info
logger.info("Python args: %s", sys.argv)
logger.info("Python interpreter: %s", python_environment())
logger.info("Operating system: %s", os_environment())
logger.info("Cpu count: %s", psutil.cpu_count())
pool_id = os.environ.get('AZ_BATCH_POOL_ID', '_test-pool-1')
node_id = os.environ.get('AZ_BATCH_NODE_ID', '_test-node-1')
# get and set event loop mode
logger.info('enabling event loop debug mode')
app_insights_key = None
if len(sys.argv) > 2:
pool_id = sys.argv[1]
node_id = sys.argv[2]
if len(sys.argv) > 3:
app_insights_key = sys.argv[3]
# create node stats manager
collector = NodeStatsCollector(
pool_id, node_id, app_insights_key=app_insights_key)
collector.init()
collector.run()
if __name__ == '__main__':
main()stderr.txt
You are using pip version 9.0.1, however version 9.0.3 is available.
You should consider upgrading via the 'python -m pip install --upgrade pip' command.
stdout.txt
Getting latest version of the Chocolatey package for download.
Getting Chocolatey from https://chocolatey.org/api/v2/package/chocolatey/0.10.9.
Downloading 7-Zip commandline tool prior to extraction.
Extracting D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall\chocolatey.zip to D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\chocInstall...
Installing chocolatey on this machine
Creating ChocolateyInstall as an environment variable (targeting 'Machine')
Setting ChocolateyInstall to 'C:\ProgramData\chocolatey'
WARNING: It's very likely you will need to close and reopen your shell
before you can use choco.
Restricting write permissions to Administrators
We are setting up the Chocolatey package repository.
The packages themselves go to 'C:\ProgramData\chocolatey\lib'
(i.e. C:\ProgramData\chocolatey\lib\yourPackageName).
A shim file for the command line goes to 'C:\ProgramData\chocolatey\bin'
and points to an executable in 'C:\ProgramData\chocolatey\lib\yourPackageName'.
Creating Chocolatey folders if they do not already exist.
WARNING: You can safely ignore errors related to missing log files when
upgrading from a version of Chocolatey less than 0.9.9.
'Batch file could not be found' is also safe to ignore.
'The system cannot find the file specified' - also safe.
chocolatey.nupkg file not installed in lib.
Attempting to locate it from bootstrapper.
PATH environment variable does not have C:\ProgramData\chocolatey\bin in it. Adding...
WARNING: Not setting tab completion: Profile file does not exist at
'D:\Users\_azbatchtask_start\Documents\WindowsPowerShell\Microsoft.PowerShell_p
rofile.ps1'.
Chocolatey (choco.exe) is now ready.
You can call choco from anywhere, command line or powershell by typing choco.
Run choco /? for a list of functions.
You may need to shut down and restart powershell and/or consoles
first prior to using choco.
Ensuring chocolatey commands are on the path
Ensuring chocolatey.nupkg is in the lib folder
Chocolatey v0.10.9
Installing the following packages:
python
By installing you accept licenses for the packages.
Progress: Downloading python3 3.6.3... 13%
Progress: Downloading python3 3.6.3... 39%
Progress: Downloading python3 3.6.3... 66%
Progress: Downloading python3 3.6.3... 93%
Progress: Downloading python3 3.6.3... 100%
Progress: Downloading python 3.6.3... 20%
Progress: Downloading python 3.6.3... 63%
Progress: Downloading python 3.6.3... 100%
python3 v3.6.3 [Approved]
python3 package files install completed. Performing other installation steps.
Downloading python3 64 bit
from 'https://www.python.org/ftp/python/3.6.3/python-3.6.3-amd64.exe'
Progress: 0% - Saving 26.78 KB of 30.16 MB
Progress: 0% - Saving 160 KB of 30.16 MB
Progress: 1% - Saving 320 KB of 30.16 MB
Progress: 1% - Saving 480 KB of 30.16 MB
Progress: 2% - Saving 640 KB of 30.16 MB
Progress: 2% - Saving 800 KB of 30.16 MB
Progress: 3% - Saving 960 KB of 30.16 MB
Progress: 3% - Saving 1.09 MB of 30.16 MB
Progress: 4% - Saving 1.25 MB of 30.16 MB
Progress: 4% - Saving 1.41 MB of 30.16 MB
Progress: 5% - Saving 1.56 MB of 30.16 MB
Progress: 5% - Saving 1.72 MB of 30.16 MB
Progress: 6% - Saving 1.88 MB of 30.16 MB
Progress: 6% - Saving 2.03 MB of 30.16 MB
Progress: 7% - Saving 2.19 MB of 30.16 MB
Progress: 7% - Saving 2.34 MB of 30.16 MB
Progress: 8% - Saving 2.5 MB of 30.16 MB
Progress: 8% - Saving 2.66 MB of 30.16 MB
Progress: 9% - Saving 2.81 MB of 30.16 MB
Progress: 9% - Saving 2.97 MB of 30.16 MB
Progress: 10% - Saving 3.13 MB of 30.16 MB
Progress: 10% - Saving 3.28 MB of 30.16 MB
Progress: 11% - Saving 3.44 MB of 30.16 MB
Progress: 11% - Saving 3.59 MB of 30.16 MB
Progress: 12% - Saving 3.75 MB of 30.16 MB
Progress: 12% - Saving 3.91 MB of 30.16 MB
Progress: 13% - Saving 4.06 MB of 30.16 MB
Progress: 13% - Saving 4.22 MB of 30.16 MB
Progress: 14% - Saving 4.38 MB of 30.16 MB
Progress: 15% - Saving 4.53 MB of 30.16 MB
Progress: 15% - Saving 4.69 MB of 30.16 MB
Progress: 16% - Saving 4.84 MB of 30.16 MB
Progress: 16% - Saving 5 MB of 30.16 MB
Progress: 17% - Saving 5.16 MB of 30.16 MB
Progress: 17% - Saving 5.31 MB of 30.16 MB
Progress: 18% - Saving 5.47 MB of 30.16 MB
Progress: 18% - Saving 5.63 MB of 30.16 MB
Progress: 19% - Saving 5.78 MB of 30.16 MB
Progress: 19% - Saving 5.94 MB of 30.16 MB
Progress: 20% - Saving 6.09 MB of 30.16 MB
Progress: 20% - Saving 6.25 MB of 30.16 MB
Progress: 21% - Saving 6.41 MB of 30.16 MB
Progress: 21% - Saving 6.56 MB of 30.16 MB
Progress: 22% - Saving 6.72 MB of 30.16 MB
Progress: 22% - Saving 6.88 MB of 30.16 MB
Progress: 23% - Saving 7.03 MB of 30.16 MB
Progress: 23% - Saving 7.19 MB of 30.16 MB
Progress: 24% - Saving 7.34 MB of 30.16 MB
Progress: 24% - Saving 7.5 MB of 30.16 MB
Progress: 25% - Saving 7.66 MB of 30.16 MB
Progress: 25% - Saving 7.81 MB of 30.16 MB
Progress: 26% - Saving 7.97 MB of 30.16 MB
Progress: 26% - Saving 8.13 MB of 30.16 MB
Progress: 27% - Saving 8.28 MB of 30.16 MB
Progress: 27% - Saving 8.44 MB of 30.16 MB
Progress: 28% - Saving 8.59 MB of 30.16 MB
Progress: 29% - Saving 8.75 MB of 30.16 MB
Progress: 29% - Saving 8.91 MB of 30.16 MB
Progress: 30% - Saving 9.06 MB of 30.16 MB
Progress: 30% - Saving 9.22 MB of 30.16 MB
Progress: 31% - Saving 9.38 MB of 30.16 MB
Progress: 31% - Saving 9.53 MB of 30.16 MB
Progress: 32% - Saving 9.69 MB of 30.16 MB
Progress: 32% - Saving 9.84 MB of 30.16 MB
Progress: 33% - Saving 10 MB of 30.16 MB
Progress: 33% - Saving 10.16 MB of 30.16 MB
Progress: 34% - Saving 10.31 MB of 30.16 MB
Progress: 34% - Saving 10.47 MB of 30.16 MB
Progress: 35% - Saving 10.63 MB of 30.16 MB
Progress: 35% - Saving 10.78 MB of 30.16 MB
Progress: 36% - Saving 10.94 MB of 30.16 MB
Progress: 36% - Saving 11.09 MB of 30.16 MB
Progress: 37% - Saving 11.25 MB of 30.16 MB
Progress: 37% - Saving 11.41 MB of 30.16 MB
Progress: 38% - Saving 11.56 MB of 30.16 MB
Progress: 38% - Saving 11.72 MB of 30.16 MB
Progress: 39% - Saving 11.88 MB of 30.16 MB
Progress: 39% - Saving 12.03 MB of 30.16 MB
Progress: 40% - Saving 12.19 MB of 30.16 MB
Progress: 40% - Saving 12.34 MB of 30.16 MB
Progress: 41% - Saving 12.5 MB of 30.16 MB
Progress: 41% - Saving 12.66 MB of 30.16 MB
Progress: 42% - Saving 12.81 MB of 30.16 MB
Progress: 43% - Saving 12.97 MB of 30.16 MB
Progress: 43% - Saving 13.13 MB of 30.16 MB
Progress: 44% - Saving 13.28 MB of 30.16 MB
Progress: 44% - Saving 13.44 MB of 30.16 MB
Progress: 45% - Saving 13.59 MB of 30.16 MB
Progress: 45% - Saving 13.75 MB of 30.16 MB
Progress: 46% - Saving 13.91 MB of 30.16 MB
Progress: 46% - Saving 14.06 MB of 30.16 MB
Progress: 47% - Saving 14.22 MB of 30.16 MB
Progress: 47% - Saving 14.38 MB of 30.16 MB
Progress: 48% - Saving 14.53 MB of 30.16 MB
Progress: 48% - Saving 14.69 MB of 30.16 MB
Progress: 49% - Saving 14.84 MB of 30.16 MB
Progress: 49% - Saving 15 MB of 30.16 MB
Progress: 50% - Saving 15.16 MB of 30.16 MB
Progress: 50% - Saving 15.31 MB of 30.16 MB
Progress: 51% - Saving 15.47 MB of 30.16 MB
Progress: 51% - Saving 15.63 MB of 30.16 MB
Progress: 52% - Saving 15.78 MB of 30.16 MB
Progress: 52% - Saving 15.94 MB of 30.16 MB
Progress: 53% - Saving 16.09 MB of 30.16 MB
Progress: 53% - Saving 16.25 MB of 30.16 MB
Progress: 54% - Saving 16.41 MB of 30.16 MB
Progress: 54% - Saving 16.56 MB of 30.16 MB
Progress: 55% - Saving 16.72 MB of 30.16 MB
Progress: 55% - Saving 16.88 MB of 30.16 MB
Progress: 56% - Saving 17.03 MB of 30.16 MB
Progress: 56% - Saving 17.19 MB of 30.16 MB
Progress: 57% - Saving 17.34 MB of 30.16 MB
Progress: 58% - Saving 17.5 MB of 30.16 MB
Progress: 58% - Saving 17.66 MB of 30.16 MB
Progress: 59% - Saving 17.81 MB of 30.16 MB
Progress: 59% - Saving 17.97 MB of 30.16 MB
Progress: 60% - Saving 18.13 MB of 30.16 MB
Progress: 60% - Saving 18.28 MB of 30.16 MB
Progress: 61% - Saving 18.44 MB of 30.16 MB
Progress: 61% - Saving 18.59 MB of 30.16 MB
Progress: 62% - Saving 18.75 MB of 30.16 MB
Progress: 62% - Saving 18.91 MB of 30.16 MB
Progress: 63% - Saving 19.06 MB of 30.16 MB
Progress: 63% - Saving 19.22 MB of 30.16 MB
Progress: 64% - Saving 19.38 MB of 30.16 MB
Progress: 64% - Saving 19.53 MB of 30.16 MB
Progress: 65% - Saving 19.69 MB of 30.16 MB
Progress: 65% - Saving 19.84 MB of 30.16 MB
Progress: 66% - Saving 20 MB of 30.16 MB
Progress: 66% - Saving 20.16 MB of 30.16 MB
Progress: 67% - Saving 20.31 MB of 30.16 MB
Progress: 67% - Saving 20.47 MB of 30.16 MB
Progress: 68% - Saving 20.63 MB of 30.16 MB
Progress: 68% - Saving 20.78 MB of 30.16 MB
Progress: 69% - Saving 20.94 MB of 30.16 MB
Progress: 69% - Saving 21.09 MB of 30.16 MB
Progress: 70% - Saving 21.25 MB of 30.16 MB
Progress: 70% - Saving 21.41 MB of 30.16 MB
Progress: 71% - Saving 21.56 MB of 30.16 MB
Progress: 72% - Saving 21.72 MB of 30.16 MB
Progress: 72% - Saving 21.88 MB of 30.16 MB
Progress: 73% - Saving 22.03 MB of 30.16 MB
Progress: 73% - Saving 22.19 MB of 30.16 MB
Progress: 74% - Saving 22.34 MB of 30.16 MB
Progress: 74% - Saving 22.5 MB of 30.16 MB
Progress: 75% - Saving 22.66 MB of 30.16 MB
Progress: 75% - Saving 22.81 MB of 30.16 MB
Progress: 76% - Saving 22.97 MB of 30.16 MB
Progress: 76% - Saving 23.13 MB of 30.16 MB
Progress: 77% - Saving 23.28 MB of 30.16 MB
Progress: 77% - Saving 23.44 MB of 30.16 MB
Progress: 78% - Saving 23.59 MB of 30.16 MB
Progress: 78% - Saving 23.75 MB of 30.16 MB
Progress: 79% - Saving 23.91 MB of 30.16 MB
Progress: 79% - Saving 24.06 MB of 30.16 MB
Progress: 80% - Saving 24.22 MB of 30.16 MB
Progress: 80% - Saving 24.38 MB of 30.16 MB
Progress: 81% - Saving 24.53 MB of 30.16 MB
Progress: 81% - Saving 24.69 MB of 30.16 MB
Progress: 82% - Saving 24.84 MB of 30.16 MB
Progress: 82% - Saving 25 MB of 30.16 MB
Progress: 83% - Saving 25.16 MB of 30.16 MB
Progress: 83% - Saving 25.31 MB of 30.16 MB
Progress: 84% - Saving 25.47 MB of 30.16 MB
Progress: 84% - Saving 25.63 MB of 30.16 MB
Progress: 85% - Saving 25.78 MB of 30.16 MB
Progress: 86% - Saving 25.94 MB of 30.16 MB
Progress: 86% - Saving 26.09 MB of 30.16 MB
Progress: 87% - Saving 26.25 MB of 30.16 MB
Progress: 87% - Saving 26.41 MB of 30.16 MB
Progress: 88% - Saving 26.56 MB of 30.16 MB
Progress: 88% - Saving 26.72 MB of 30.16 MB
Progress: 89% - Saving 26.88 MB of 30.16 MB
Progress: 89% - Saving 27.03 MB of 30.16 MB
Progress: 90% - Saving 27.19 MB of 30.16 MB
Progress: 90% - Saving 27.34 MB of 30.16 MB
Progress: 91% - Saving 27.5 MB of 30.16 MB
Progress: 91% - Saving 27.66 MB of 30.16 MB
Progress: 92% - Saving 27.81 MB of 30.16 MB
Progress: 92% - Saving 27.97 MB of 30.16 MB
Progress: 93% - Saving 28.13 MB of 30.16 MB
Progress: 93% - Saving 28.28 MB of 30.16 MB
Progress: 94% - Saving 28.44 MB of 30.16 MB
Progress: 94% - Saving 28.59 MB of 30.16 MB
Progress: 95% - Saving 28.75 MB of 30.16 MB
Progress: 95% - Saving 28.91 MB of 30.16 MB
Progress: 96% - Saving 29.06 MB of 30.16 MB
Progress: 96% - Saving 29.22 MB of 30.16 MB
Progress: 97% - Saving 29.38 MB of 30.16 MB
Progress: 97% - Saving 29.53 MB of 30.16 MB
Progress: 98% - Saving 29.69 MB of 30.16 MB
Progress: 98% - Saving 29.84 MB of 30.16 MB
Progress: 99% - Saving 30 MB of 30.16 MB
Progress: 100% - Saving 30.16 MB of 30.16 MB
Progress: 100% - Completed download of D:\Users\_azbatchtask_start\AppData\Local\Temp\chocolatey\python3\3.6.3\python-3.6.3-amd64.exe (30.16 MB).
Download of python-3.6.3-amd64.exe (30.16 MB) completed.
Hashes match.
Installing python3...
python3 has been installed.
Installed to 'C:\Python36'
python3 can be automatically uninstalled.
Environment Vars (like PATH) have changed. Close/reopen your shell to
see the changes (or in powershell/cmd.exe just type `refreshenv`).
The install of python3 was successful.
Software installed as 'EXE', install location is likely default.
python v3.6.3 [Approved]
python package files install completed. Performing other installation steps.
The install of python was successful.
Software install location not explicitly set, could be in package or
default install location if installer.
Chocolatey installed 2/2 packages.
See the log for details (C:\ProgramData\chocolatey\logs\chocolatey.log).
Current path: C:\Python36\Scripts\;C:\Python36\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\ProgramData\chocolatey\bin;;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin;C:\Anaconda;C:\Anaconda\Library\mingw-w64\bin;C:\Anaconda\Library\usr\bin;C:\Anaconda\Library\bin;C:\Anaconda\Scripts;C:\Program Files\Microsoft MPI\Bin\;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\130\DTS\Binn\;C:\Program Files\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\130\Tools\Binn\;C:\Program Files\Microsoft SQL Server\120\Tools\Binn\;C:\Program Files\Microsoft\Web Platform Installer\;C:\Program Files (x86)\Microsoft SDKs\Azure\CLI\wbin;C:\dsvm\tools\mxnet\\lib;C:\dsvm\tools\mxnet\\3rdparty\cudnn\bin;C:\dsvm\tools\mxnet\\3rdparty\cudart;C:\dsvm\tools\mxnet\\3rdparty\vc;C:\dsvm\tools\mxnet\\3rdparty\gnuwin;C:\dsvm\tools\mxnet\\3rdparty\openblas\bin;C:\dsvm\tools\bin;C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;C:\dsvm\tools\DataMovement\ADL;C:\dsvm\tools\DataMovement\DocumentDB;C:\Program Files\Microsoft\R Server\R_SERVER\bin\x64;C:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin;c:\dsvm\tools\cntk2\cntk;C:\Program Files (x86)\Pandoc\;c:\Program Files\Zulu\zulu8.17.0.3-jdk8.0.102-win_x64\jre\bin\server;C:\JuliaPro-0.5.0.2\Julia-0.5.0\bin;C:\Program Files\dotnet\;C:\Program Files\nodejs\;C:\Program Files (x86)\Microsoft SQL Server\Client SDK\ODBC\130\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\140\Tools\Binn\ManagementStudio\;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;D:\batch\tasks\shared;D:\batch\tasks\startup\wd
Python version:
Python 3.6.3
Collecting psutil
Downloading psutil-5.4.3-cp36-cp36m-win_amd64.whl (226kB)
Collecting python-dateutil
Downloading python_dateutil-2.7.2-py2.py3-none-any.whl (212kB)
Collecting applicationinsights
Downloading applicationinsights-0.11.2.tar.gz (45kB)
Collecting six>=1.5 (from python-dateutil)
Downloading six-1.11.0-py2.py3-none-any.whl
Installing collected packages: psutil, six, python-dateutil, applicationinsights
Running setup.py install for applicationinsights: started
Running setup.py install for applicationinsights: finished with status 'done'
Successfully installed applicationinsights-0.11.2 psutil-5.4.3 python-dateutil-2.7.2 six-1.11.0
Downloading nodestats.py
Starting App insights background process in D:\batch\tasks\startup\wd
TaskPath TaskName
-------- --------
\ batchappinsights
\ batchappinsights
from batch-insights.
timotheeguerin
commented on May 20, 2024
hhm, you said you are using dsvm? It is working for me on a normal windows pool. Maybe the dsvm now has 2 version of python installed and they conflict with each other.
from batch-insights.
jeroenwo
commented on May 20, 2024
Yes, it already has Python installed. Do I have to copy the PowerShell script and comment out the installation of Python (choco install -y python --version 3.6.3)? Or maybe add a parameter to skip the installation in the original script?
from batch-insights.
timotheeguerin
commented on May 20, 2024
So after trying a few things, problem is: script install 3.6 then install dependencies for that python version. But then the backgorund task is started as the system user which use the anaconda version of python.
I am trying to hard code the script to use the version of python I already installed.
Other options could be to check if python is already installed and if so use it. But then could get into some other issues of python dependencies issues and wrong version of python.
Could also have as you mentioned a flag not to install python
from batch-insights.
Related Issues (20)
- Hitting chocolatey rate limits HOT 5
- CPU stats on windows giving more than 100%
- GPU monitoring support
- Crash in v1.1.0 using standard_d2s_v3 VMs HOT 6
- Pre-aggregate metrics to reduce cost
- New configuration system to allow for picking which metrics to upload
- Batch Insight crash if instrumentation key is missing HOT 8
- No Metrics Showing HOT 27
- batch insights crashes with "invalid memory address or nil pointer dereference" HOT 1
- Register-ScheduledTask : Access is denied
- How Do I view Batch-Insights metrics from the Azure Portal? HOT 1
- CPU consumptio showing different values on Batch explorer and App insights & Lock issue for node post every job run.
- Could not get lock /var/lib/dpkg/lock-frontend HOT 5
- "latest" tag for generic release URL HOT 1
- I'm stupid but HOT 1
- batch-insights running but no metrics available in both BatchExplorer and Application Insights
- Typo in config.go
- No metrics shown in batch explorer and app insights HOT 1
- Ambiguous wording in Readme
- This repo is missing important files
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from batch-insights.