diff --git a/experiments/pycom_ping/Dockerfile b/experiments/pycom_ping/Dockerfile new file mode 100644 index 0000000..8a5d714 --- /dev/null +++ b/experiments/pycom_ping/Dockerfile @@ -0,0 +1,22 @@ +FROM monroe/base + +MAINTAINER Jonas.Karlsson@kau.se + +COPY files/* /opt/monroe/ + + + +############## Install pyserial #################### +ENV APT_OPTS -y --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends --no-install-suggests --allow-unauthenticated +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install ${APT_OPTS} \ + python3-serial \ + && apt-get update ${APT_OPTS} --fix-missing \ + # Cleanup + && apt-get clean ${APT_OPTS} \ + && apt-get autoremove ${APT_OPTS} \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc /usr/share/man /usr/share/locale /var/cache/debconf/*-old + +WORKDIR /opt/monroe +ENTRYPOINT ["dumb-init", "--", "/usr/bin/python3", "/opt/monroe/pycom_ping.py"] diff --git a/experiments/pycom_ping/build.sh b/experiments/pycom_ping/build.sh new file mode 100755 index 0000000..544fab5 --- /dev/null +++ b/experiments/pycom_ping/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +CONTAINER=${DIR##*/} + +docker pull monroe/base +docker build --rm --no-cache -t ${CONTAINER} . && echo "Finished building ${CONTAINER}" diff --git a/experiments/pycom_ping/files/fipy-helper-functions.py b/experiments/pycom_ping/files/fipy-helper-functions.py new file mode 100644 index 0000000..6069edf --- /dev/null +++ b/experiments/pycom_ping/files/fipy-helper-functions.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +# Author: Mohammad Rajiullah +# Date: November 2019 +# License: + +import utime +import uselect +import uctypes +import usocket +import ustruct +import machine +from network import LTE +import time +import usocket +import json + +def pprint(msg): + print(msg, end = '') + +def send_at_cmd_pretty(cmd,lte): + """Prettify AT command output""" + response = lte.send_at_cmd(cmd).split('\r\n') + for line in response: + print(line) + +def set_dns(servers=['8.8.8.8','4.4.4.4']): + """Setting dns for the NB setting""" + i=0 + for server in servers: + usocket.dnsserver(i,server) + i+=1 + pprint(usocket.dnsserver()) + +def set_nbiot(apn="lpwa.telia.iot", band=None, type=LTE.IP, wait=10): + """Setting the NB-IOT connection""" + factor=0.25 + ipdown = True + lte = LTE() + lte.reset() + lte.attach(apn=apn,band=band, type=type) + i=0 + while not lte.isattached() and i < wait/factor: + time.sleep(factor) + i+=1 + if lte.isattached(): + print("Attached to a LTE network (took {} s)".format(i*factor)) + send_at_cmd_pretty('AT+CGCONTRDP',lte) + lte.connect() # start a data session and obtain an IP address + i=0 + while lte.isattached() and not lte.isconnected() and i < wait/factor: + time.sleep(factor) + i+=1 + if lte.isconnected(): + print("Connected to a LTE network (took {} s)".format(i*factor)) + i=0 + while ipdown and i < wait/factor: + ipdown = False + try: + usocket.getaddrinfo("8.8.8.8", 80) + except Exception as e: + ipdown = True + time.sleep(factor) + i+=1 + if not ipdown: + print("Interface up (took {} s)".format(i*factor)) + pprint("OK") + else: + pprint("Failed") + +def get_connection_status(): + """Getting NB-IOT connection status""" + lte = LTE() + if not lte.isattached(): + pprint("Not attached") + elif not lte.isconnected(): + pprint("Not connected") + else: + try: + usocket.getaddrinfo("8.8.8.8", 80) + except Exception as e: + pprint("Interface down") + pprint("Connected") + +def get_iccid(wait=10): + factor=0.25 + lte = LTE() + if lte.isconnected(): + lte.pppsuspend() + pprint(lte.iccid()) + lte.pppresume() + else: + i=0 + while not lte.iccid() and i < wait/factor: + time.sleep(factor) + i+=1 + pprint(lte.iccid()) + +def checksum(data): + if len(data) & 0x1: # Odd number of bytes + data += b'\0' + cs = 0 + for pos in range(0, len(data), 2): + b1 = data[pos] + b2 = data[pos + 1] + cs += (b1 << 8) + b2 + while cs >= 0x10000: + cs = (cs & 0xffff) + (cs >> 16) + cs = ~cs & 0xffff + return cs + +def ping(host, count=4, timeout=5000, interval=10, quiet=True, size=64): + """Ping implementation for upython""" + assert size >= 16, "pkt size too small" + pkt = b'Q'*size + pkt_desc = { + "type": uctypes.UINT8 | 0, + "code": uctypes.UINT8 | 1, + "checksum": uctypes.UINT16 | 2, + "id": uctypes.UINT16 | 4, + "seq": uctypes.INT16 | 6, + "timestamp": uctypes.UINT64 | 8, + } # packet header descriptor + h = uctypes.struct(uctypes.addressof(pkt), pkt_desc, uctypes.BIG_ENDIAN) + h.type = 8 # ICMP_ECHO_REQUEST + h.code = 0 + h.checksum = 0 + #h.id = urandom.randint(0, 65535) + h.id = machine.rng() & 0xffff + h.seq = 1 + + # init socket + #sock = usocket.socket(usocket.AF_INET, usocket.SOCK_RAW, 1) + sock = usocket.socket(usocket.AF_INET, 3, 1) + sock.setblocking(0) + sock.settimeout(timeout/1000) + addr = usocket.getaddrinfo(host, 1)[0][-1][0] # ip address + sock.connect((addr, 1)) + not quiet and pprint("PING %s (%s): %u data bytes" % (host, addr, len(pkt))) + + seqs = list(range(1, count+1)) # [1,2,...,count] + ping_results=[] + c = 1 + t = 0 + n_trans = 0 + n_recv = 0 + finish = False + results = [] + while t < timeout: + if t==interval and c<=count: + # send packet + h.checksum = 0 + h.seq = c + h.timestamp = utime.ticks_us() + h.checksum = checksum(pkt) + if sock.send(pkt) == size: + n_trans += 1 + t = 0 # reset timeout + else: + seqs.remove(c) + c += 1 + + # recv packet + while 1: + socks, _, _ = uselect.select([sock], [], [], 0) + if socks: + resp = socks[0].recv(4096) + resp_mv = memoryview(resp) + h2 = uctypes.struct(uctypes.addressof(resp_mv[20:]), pkt_desc, uctypes.BIG_ENDIAN) + # TODO: validate checksum (optional) + seq = h2.seq + if h2.type==0 and h2.id==h.id and (seq in seqs): # 0: ICMP_ECHO_REPLY + #t_elasped = (utime.ticks_us()-h2.timestamp) / 1000 + t_elasped = abs(utime.ticks_diff(utime.ticks_us(),h2.timestamp) / 1000) + ttl = ustruct.unpack('!B', resp_mv[8:9])[0] # time-to-live + n_recv += 1 + not quiet and pprint("%u bytes from %s: icmp_seq=%u, ttl=%u, time=%f ms" % (len(resp), addr, seq, ttl, t_elasped)) + results.append({ "from": addr, + "bytes": len(resp), + "seq": seq, + "ttl": ttl, + "time": t_elasped + }) + ping_results.append(t_elasped) + seqs.remove(seq) + if len(seqs) == 0: + finish = True + break + else: + break + + if finish: + break + + utime.sleep_ms(1) + t += 1 + + # close + sock.close() + ret = (n_trans, n_recv) + not quiet and pprint("%u packets transmitted, %u packets received" % (n_trans, n_recv)) + + result = { "to": host, + "addr": addr, + "bytes": len(pkt), + "n_trans": n_trans, + "n_recv": n_recv, + "rtts" : ping_results, + "pkts": results + } + pprint (json.dumps(result)) diff --git a/experiments/pycom_ping/files/monroe_exporter.py b/experiments/pycom_ping/files/monroe_exporter.py new file mode 100644 index 0000000..0102221 --- /dev/null +++ b/experiments/pycom_ping/files/monroe_exporter.py @@ -0,0 +1,192 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Authors: Jonas Karlsson +# Date: Dec 2015 +# License: GNU General Public License v3 +# Developed for use by the EU H2020 MONROE project + +# CODENAME : Unicorn +"""Export json string objects to file.""" +import json +import time +from threading import Semaphore, Timer +import os +import tempfile +import argparse +import textwrap +import sys + +CMD_NAME = os.path.basename(__file__) +TEMP_FILE_NAME = None +TEMP_FILE = None +FILE_SEMA = Semaphore() +JSON_STORE = [] +INITIALIZED = False +DEBUG = False + + +def initalize(interval, outdir="/monroe/results/"): + """Bootstrapping timed saves.""" + global INITIALIZED + if not INITIALIZED: + _timed_move_to_output_(outdir, interval) + INITIALIZED = True + + +def save_output(msg, outdir="/monroe/results/"): + """Save the msg.""" + global FILE_SEMA, JSON_STORE + with FILE_SEMA: + JSON_STORE.append(msg) + + if not INITIALIZED: + _timed_move_to_output_(outdir, -1) + + +def _timed_move_to_output_(outdir, interval): + """Called every interval seconds and move the file to the output directory. + + For later transfer to the remote repository. + """ + global JSON_STORE, FILE_SEMA + + # Grab the file semaphore so we do not read the CACHE while the other + # thread is updating it + with FILE_SEMA: + if len(JSON_STORE) > 0: + # Create a name for the file from the first msg in the list + # and check for obligatory variables + nodeid = JSON_STORE[0]['NodeId'] + dataid = JSON_STORE[0]['DataId'] + dataversion = JSON_STORE[0]['DataVersion'] + dest_name = outdir + "{}_{}_{}_{}.json".format(nodeid, + dataid, + dataversion, + time.time()) + + # A 'atomic copy' + # copy contents of tmp file to outdir + # and then rename (atomic operation) to final filename + statv = os.statvfs(outdir) + # Only save file if more than 1 Mbyte free + if statv.f_bfree*statv.f_bsize > 1048576: + try: + tmp_dest_name = None + with tempfile.NamedTemporaryFile(mode='w', + dir=outdir, + delete=False,) as tmp_dest: + tmp_dest_name = tmp_dest.name + + for msg in JSON_STORE: + try: + msg['NodeId'] + msg['DataId'] + msg['DataVersion'] + msg['Timestamp'] + msg['SequenceNumber'] + if DEBUG: + print(json.dumps(msg)) + else: + json.dump(msg, tmp_dest) + except Exception as e: + errormsg = ("Error: {} {}," + "skipping this message " + "in {}({})").format(e, + msg, + tmp_dest_name, + dest_name) + print(errormsg) + continue + + tmp_dest.flush() + os.fsync(tmp_dest.fileno()) + + if (os.stat(tmp_dest_name).st_size > 0): + # atomic rename of /outdir/tmpXXXX -> /outdir/yyy.json + os.rename(tmp_dest_name, dest_name) + os.chmod(dest_name, 0o644) + JSON_STORE = [] + # print "Info: Moved {} -> {}".format(tmp_dest_name, + # dest_name) + else: + os.unlink(tmp_dest_name) + except Exception as e: + log_str = "Error: {} {} : {}".format(dest_name, + tmp_dest_name, + e) + try: + os.unlink(tmp_dest_name) + except Exception as e: + pass + print(log_str) + else: + # We have too little space left on outdir + log_str = "Error: Out of disk space: {} ".format(dest_name) + print(log_str) + + if interval > 1: + # ..Reschedule me in interval seconds + t = Timer(interval, lambda: _timed_move_to_output_(outdir, interval)) + t.daemon = True # Will stop with the main program + t.start() + + +def create_arg_parser(): + """Create a argument parser and return it.""" + parser = argparse.ArgumentParser( + prog=CMD_NAME, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent(''' + Save experiment/metadata output for later transport + to repository''')) + parser.add_argument('--msg', + required=True, + help=("Experiment/Metadata msg(in JSON format)" + "Obligatory keys: NodeId, DataId, DataVersion, " + "TimeStamp, SequenceNumber")) + parser.add_argument('--outdir', + metavar='DIR', + default="/monroe/results/", + help=("Directory to save the results to" + "(default /monroe/results/)")) + parser.add_argument('--debug', + action="store_true", + help="Do not save files") + parser.add_argument('-v', '--version', + action="version", + version="%(prog)s 1.0") + return parser + + +if __name__ == '__main__': + parser = create_arg_parser() + args = parser.parse_args() + DEBUG = args.debug + try: + jsonmsg = json.loads(args.msg) + jsonmsg['NodeId'] + jsonmsg['DataId'] + jsonmsg['DataVersion'] + jsonmsg['Timestamp'] + jsonmsg['SequenceNumber'] + except Exception as e: + errormsg = ("Error: called from commandline with" + " invalid JSON got {} : {}").format(args.msg, e) + print(errormsg) + sys.exit(1) + + outdir = str(args.outdir) + if not outdir.endswith('/'): + print("Info: Corrected missing last / in outdir={}".format(outdir)) + outdir += '/' + + if DEBUG: + print(("Debug mode: will not insert any posts\n" + "Info and Statements are printed to stdout\n" + "{} called with variables \noutdir={}" + " \nmsg={} \njson={}").format(CMD_NAME, + outdir, + args.msg, + jsonmsg)) + save_output(jsonmsg, outdir) diff --git a/experiments/pycom_ping/files/pyboard.py b/experiments/pycom_ping/files/pyboard.py new file mode 100755 index 0000000..c32fb00 --- /dev/null +++ b/experiments/pycom_ping/files/pyboard.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2014-2019 Damien P. George +# Copyright (c) 2017 Paul Sokolovsky +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" +pyboard interface + +This module provides the Pyboard class, used to communicate with and +control a MicroPython device over a communication channel. Both real +boards and emulated devices (e.g. running in QEMU) are supported. +Various communication channels are supported, including a serial +connection, telnet-style network connection, external process +connection. + +Example usage: + + import pyboard + pyb = pyboard.Pyboard('/dev/ttyACM0') + +Or: + + pyb = pyboard.Pyboard('192.168.1.1') + +Then: + + pyb.enter_raw_repl() + pyb.exec('import pyb') + pyb.exec('pyb.LED(1).on()') + pyb.exit_raw_repl() + +Note: if using Python2 then pyb.exec must be written as pyb.exec_. +To run a script from the local machine on the board and print out the results: + + import pyboard + pyboard.execfile('test.py', device='/dev/ttyACM0') + +This script can also be run directly. To execute a local script, use: + + ./pyboard.py test.py + +Or: + + python pyboard.py test.py + +""" + +import sys +import time +import os + +try: + stdout = sys.stdout.buffer +except AttributeError: + # Python2 doesn't have buffer attr + stdout = sys.stdout + +def stdout_write_bytes(b): + b = b.replace(b"\x04", b"") + stdout.write(b) + stdout.flush() + +class PyboardError(Exception): + pass + +class TelnetToSerial: + def __init__(self, ip, user, password, read_timeout=None): + self.tn = None + import telnetlib + self.tn = telnetlib.Telnet(ip, timeout=15) + self.read_timeout = read_timeout + if b'Login as:' in self.tn.read_until(b'Login as:', timeout=read_timeout): + self.tn.write(bytes(user, 'ascii') + b"\r\n") + + if b'Password:' in self.tn.read_until(b'Password:', timeout=read_timeout): + # needed because of internal implementation details of the telnet server + time.sleep(0.2) + self.tn.write(bytes(password, 'ascii') + b"\r\n") + + if b'for more information.' in self.tn.read_until(b'Type "help()" for more information.', timeout=read_timeout): + # login successful + from collections import deque + self.fifo = deque() + return + + raise PyboardError('Failed to establish a telnet connection with the board') + + def __del__(self): + self.close() + + def close(self): + if self.tn: + self.tn.close() + + def read(self, size=1): + while len(self.fifo) < size: + timeout_count = 0 + data = self.tn.read_eager() + if len(data): + self.fifo.extend(data) + timeout_count = 0 + else: + time.sleep(0.25) + if self.read_timeout is not None and timeout_count > 4 * self.read_timeout: + break + timeout_count += 1 + + data = b'' + while len(data) < size and len(self.fifo) > 0: + data += bytes([self.fifo.popleft()]) + return data + + def write(self, data): + self.tn.write(data) + return len(data) + + def inWaiting(self): + n_waiting = len(self.fifo) + if not n_waiting: + data = self.tn.read_eager() + self.fifo.extend(data) + return len(data) + else: + return n_waiting + + +class ProcessToSerial: + "Execute a process and emulate serial connection using its stdin/stdout." + + def __init__(self, cmd): + import subprocess + self.subp = subprocess.Popen(cmd, bufsize=0, shell=True, preexec_fn=os.setsid, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Initially was implemented with selectors, but that adds Python3 + # dependency. However, there can be race conditions communicating + # with a particular child process (like QEMU), and selectors may + # still work better in that case, so left inplace for now. + # + #import selectors + #self.sel = selectors.DefaultSelector() + #self.sel.register(self.subp.stdout, selectors.EVENT_READ) + + import select + self.poll = select.poll() + self.poll.register(self.subp.stdout.fileno()) + + def close(self): + import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) + + def read(self, size=1): + data = b"" + while len(data) < size: + data += self.subp.stdout.read(size - len(data)) + return data + + def write(self, data): + self.subp.stdin.write(data) + return len(data) + + def inWaiting(self): + #res = self.sel.select(0) + res = self.poll.poll(0) + if res: + return 1 + return 0 + + +class ProcessPtyToTerminal: + """Execute a process which creates a PTY and prints slave PTY as + first line of its output, and emulate serial connection using + this PTY.""" + + def __init__(self, cmd): + import subprocess + import re + import serial + self.subp = subprocess.Popen(cmd.split(), bufsize=0, shell=False, preexec_fn=os.setsid, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pty_line = self.subp.stderr.readline().decode("utf-8") + m = re.search(r"/dev/pts/[0-9]+", pty_line) + if not m: + print("Error: unable to find PTY device in startup line:", pty_line) + self.close() + sys.exit(1) + pty = m.group() + # rtscts, dsrdtr params are to workaround pyserial bug: + # http://stackoverflow.com/questions/34831131/pyserial-does-not-play-well-with-virtual-port + self.ser = serial.Serial(pty, interCharTimeout=1, rtscts=True, dsrdtr=True) + + def close(self): + import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) + + def read(self, size=1): + return self.ser.read(size) + + def write(self, data): + return self.ser.write(data) + + def inWaiting(self): + return self.ser.inWaiting() + + +class Pyboard: + def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0): + if device.startswith("exec:"): + self.serial = ProcessToSerial(device[len("exec:"):]) + elif device.startswith("execpty:"): + self.serial = ProcessPtyToTerminal(device[len("qemupty:"):]) + elif device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3: + # device looks like an IP address + self.serial = TelnetToSerial(device, user, password, read_timeout=10) + else: + import serial + delayed = False + for attempt in range(wait + 1): + try: + self.serial = serial.Serial(device, baudrate=baudrate, interCharTimeout=1) + break + except (OSError, IOError): # Py2 and Py3 have different errors + if wait == 0: + continue + if attempt == 0: + sys.stdout.write('Waiting {} seconds for pyboard '.format(wait)) + delayed = True + time.sleep(1) + sys.stdout.write('.') + sys.stdout.flush() + else: + if delayed: + print('') + raise PyboardError('failed to access ' + device) + if delayed: + print('') + + def close(self): + self.serial.close() + + def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): + # if data_consumer is used then data is not accumulated and the ending must be 1 byte long + assert data_consumer is None or len(ending) == 1 + + data = self.serial.read(min_num_bytes) + if data_consumer: + data_consumer(data) + timeout_count = 0 + while True: + if data.endswith(ending): + break + elif self.serial.inWaiting() > 0: + new_data = self.serial.read(1) + if data_consumer: + data_consumer(new_data) + data = new_data + else: + data = data + new_data + timeout_count = 0 + else: + timeout_count += 1 + if timeout is not None and timeout_count >= 100 * timeout: + break + time.sleep(0.01) + return data + + def enter_raw_repl(self): + self.serial.write(b'\r\x03\x03') # ctrl-C twice: interrupt any running program + + # flush input (without relying on serial.flushInput()) + n = self.serial.inWaiting() + while n > 0: + self.serial.read(n) + n = self.serial.inWaiting() + + self.serial.write(b'\r\x01') # ctrl-A: enter raw REPL + data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n>') + if not data.endswith(b'raw REPL; CTRL-B to exit\r\n>'): + print(data) + raise PyboardError('could not enter raw repl') + + self.serial.write(b'\x04') # ctrl-D: soft reset + data = self.read_until(1, b'soft reboot\r\n') + if not data.endswith(b'soft reboot\r\n'): + print(data) + raise PyboardError('could not enter raw repl') + # By splitting this into 2 reads, it allows boot.py to print stuff, + # which will show up after the soft reboot and before the raw REPL. + data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n') + if not data.endswith(b'raw REPL; CTRL-B to exit\r\n'): + print(data) + raise PyboardError('could not enter raw repl') + + def exit_raw_repl(self): + self.serial.write(b'\r\x02') # ctrl-B: enter friendly REPL + + def follow(self, timeout, data_consumer=None): + # wait for normal output + data = self.read_until(1, b'\x04', timeout=timeout, data_consumer=data_consumer) + if not data.endswith(b'\x04'): + raise PyboardError('timeout waiting for first EOF reception') + data = data[:-1] + + # wait for error output + data_err = self.read_until(1, b'\x04', timeout=timeout) + if not data_err.endswith(b'\x04'): + raise PyboardError('timeout waiting for second EOF reception') + data_err = data_err[:-1] + + # return normal and error output + return data, data_err + + def exec_raw_no_follow(self, command): + if isinstance(command, bytes): + command_bytes = command + else: + command_bytes = bytes(command, encoding='utf8') + + # check we have a prompt + data = self.read_until(1, b'>') + if not data.endswith(b'>'): + raise PyboardError('could not enter raw repl') + + # write command + for i in range(0, len(command_bytes), 256): + self.serial.write(command_bytes[i:min(i + 256, len(command_bytes))]) + time.sleep(0.01) + self.serial.write(b'\x04') + + # check if we could exec command + data = self.serial.read(2) + if data != b'OK': + raise PyboardError('could not exec command (response: %r)' % data) + + def exec_raw(self, command, timeout=10, data_consumer=None): + self.exec_raw_no_follow(command); + return self.follow(timeout, data_consumer) + + def eval(self, expression): + ret = self.exec_('print({})'.format(expression)) + ret = ret.strip() + return ret + + def exec_(self, command, data_consumer=None): + ret, ret_err = self.exec_raw(command, data_consumer=data_consumer) + if ret_err: + raise PyboardError('exception', ret, ret_err) + return ret + + def execfile(self, filename): + with open(filename, 'rb') as f: + pyfile = f.read() + return self.exec_(pyfile) + + def get_time(self): + t = str(self.eval('pyb.RTC().datetime()'), encoding='utf8')[1:-1].split(', ') + return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6]) + + def fs_ls(self, src): + cmd = "import uos\nfor f in uos.ilistdir(%s):\n" \ + " print('{:12} {}{}'.format(f[3]if len(f)>3 else 0,f[0],'/'if f[1]&0x4000 else ''))" % \ + (("'%s'" % src) if src else '') + self.exec_(cmd, data_consumer=stdout_write_bytes) + + def fs_cat(self, src, chunk_size=256): + cmd = "with open('%s') as f:\n while 1:\n" \ + " b=f.read(%u)\n if not b:break\n print(b,end='')" % (src, chunk_size) + self.exec_(cmd, data_consumer=stdout_write_bytes) + + def fs_get(self, src, dest, chunk_size=256): + self.exec_("f=open('%s','rb')\nr=f.read" % src) + with open(dest, 'wb') as f: + while True: + data = bytearray() + self.exec_("print(r(%u))" % chunk_size, data_consumer=lambda d:data.extend(d)) + assert data.endswith(b'\r\n\x04') + data = eval(str(data[:-3], 'ascii')) + if not data: + break + f.write(data) + self.exec_("f.close()") + + def fs_put(self, src, dest, chunk_size=256): + self.exec_("f=open('%s','wb')\nw=f.write" % dest) + with open(src, 'rb') as f: + while True: + data = f.read(chunk_size) + if not data: + break + if sys.version_info < (3,): + self.exec_('w(b' + repr(data) + ')') + else: + self.exec_('w(' + repr(data) + ')') + self.exec_("f.close()") + + def fs_mkdir(self, dir): + self.exec_("import uos\nuos.mkdir('%s')" % dir) + + def fs_rmdir(self, dir): + self.exec_("import uos\nuos.rmdir('%s')" % dir) + + def fs_rm(self, src): + self.exec_("import uos\nuos.remove('%s')" % src) + +# in Python2 exec is a keyword so one must use "exec_" +# but for Python3 we want to provide the nicer version "exec" +setattr(Pyboard, "exec", Pyboard.exec_) + +def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', password='python'): + pyb = Pyboard(device, baudrate, user, password) + pyb.enter_raw_repl() + output = pyb.execfile(filename) + stdout_write_bytes(output) + pyb.exit_raw_repl() + pyb.close() + +def filesystem_command(pyb, args): + def fname_remote(src): + if src.startswith(':'): + src = src[1:] + return src + def fname_cp_dest(src, dest): + src = src.rsplit('/', 1)[-1] + if dest is None or dest == '': + dest = src + elif dest == '.': + dest = './' + src + elif dest.endswith('/'): + dest += src + return dest + + cmd = args[0] + args = args[1:] + try: + if cmd == 'cp': + srcs = args[:-1] + dest = args[-1] + if srcs[0].startswith('./') or dest.startswith(':'): + op = pyb.fs_put + fmt = 'cp %s :%s' + dest = fname_remote(dest) + else: + op = pyb.fs_get + fmt = 'cp :%s %s' + for src in srcs: + src = fname_remote(src) + dest2 = fname_cp_dest(src, dest) + print(fmt % (src, dest2)) + op(src, dest2) + else: + op = {'ls': pyb.fs_ls, 'cat': pyb.fs_cat, 'mkdir': pyb.fs_mkdir, + 'rmdir': pyb.fs_rmdir, 'rm': pyb.fs_rm}[cmd] + if cmd == 'ls' and not args: + args = [''] + for src in args: + src = fname_remote(src) + print('%s :%s' % (cmd, src)) + op(src) + except PyboardError as er: + print(str(er.args[2], 'ascii')) + pyb.exit_raw_repl() + pyb.close() + sys.exit(1) + +def main(): + import argparse + cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.') + cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') + cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') + cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') + cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') + cmd_parser.add_argument('-c', '--command', help='program passed in as string') + cmd_parser.add_argument('-w', '--wait', default=0, type=int, help='seconds to wait for USB connected board to become available') + cmd_parser.add_argument('--follow', action='store_true', help='follow the output after running the scripts [default if no scripts given]') + cmd_parser.add_argument('-f', '--filesystem', action='store_true', help='perform a filesystem action') + cmd_parser.add_argument('files', nargs='*', help='input files') + args = cmd_parser.parse_args() + + # open the connection to the pyboard + try: + pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait) + except PyboardError as er: + print(er) + sys.exit(1) + + # run any command or file(s) + if args.command is not None or args.filesystem or len(args.files): + # we must enter raw-REPL mode to execute commands + # this will do a soft-reset of the board + try: + pyb.enter_raw_repl() + except PyboardError as er: + print(er) + pyb.close() + sys.exit(1) + + def execbuffer(buf): + try: + ret, ret_err = pyb.exec_raw(buf, timeout=None, data_consumer=stdout_write_bytes) + except PyboardError as er: + print(er) + pyb.close() + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) + if ret_err: + pyb.exit_raw_repl() + pyb.close() + stdout_write_bytes(ret_err) + sys.exit(1) + + # do filesystem commands, if given + if args.filesystem: + filesystem_command(pyb, args.files) + args.files.clear() + + # run the command, if given + if args.command is not None: + execbuffer(args.command.encode('utf-8')) + + # run any files + for filename in args.files: + with open(filename, 'rb') as f: + pyfile = f.read() + execbuffer(pyfile) + + # exiting raw-REPL just drops to friendly-REPL mode + pyb.exit_raw_repl() + + # if asked explicitly, or no files given, then follow the output + if args.follow or (args.command is None and not args.filesystem and len(args.files) == 0): + try: + ret, ret_err = pyb.follow(timeout=None, data_consumer=stdout_write_bytes) + except PyboardError as er: + print(er) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) + if ret_err: + pyb.close() + stdout_write_bytes(ret_err) + sys.exit(1) + + # close the connection to the pyboard + pyb.close() + +if __name__ == "__main__": + main() diff --git a/experiments/pycom_ping/files/pycom_ping.py b/experiments/pycom_ping/files/pycom_ping.py new file mode 100644 index 0000000..3b7c791 --- /dev/null +++ b/experiments/pycom_ping/files/pycom_ping.py @@ -0,0 +1,189 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Author: Jonas Karlsson +# Date: Nov 2019 +# License: GNU General Public License v3 +# Developed for use by the EU H2020 MONROE project + +""" +Simple wrapper to run ping on a pycom device using nb-iot. + +The script will run a limited set of times (default 4) on the specified +interface, count < 1 it will run forever. +All default values are configurable from the scheduler. +The output will be formated into a json object suitable for storage in the +MONROE db. +""" +import pyboard +import json +import re +import time + +# Configuration +CONFIGFILE = '/monroe/config' + +# Default values (overwritable from the scheduler) +EXPCONFIG = { + "guid": "no.guid.in.config.file", # Should be overridden by scheduler + "nodeid": "fake.nodeid", + "server": "8.8.8.8", # ping target + "interval": 5000, # time in milliseconds between successive packets + "dataversion": 1, + "count": 4, + "size":56, + "device": "/dev/pycom/board0", +# "device": "/dev/tty.usbserial-DQ00DARH", + "apn": "lpwa.telia.iot", + "type": "LTE.IP", #LTE.IP or LTE.IPV4V6 defualt LTE.IP + "band": None, # scans all bands + "dataid": "MONROE.EXP.NBPING", + "export_interval": 5.0, + "verbosity": 2, # 0 = "Mute", 1=error, 2=Information, 3=verbose + "resultdir": "/monroe/results/", + "dns_servers": ['8.8.8.8', '4.4.4.4'], + "DEBUG": False + } + +def pyexec(cmd=None, pyb=None): + """Returns a translated str""" + if cmd and pyb: + return pyb.exec(cmd).decode('utf-8').replace('/r/n', '/n') + else: + return "" + +if __name__ == '__main__': + """The main thread.""" + + # Try to get the experiment config as provided by the scheduler + try: + with open(CONFIGFILE) as configfd: + EXPCONFIG.update(json.load(configfd)) + except Exception as e: + print ("Cannot retrive expconfig {}".format(e)) + EXPCONFIG['DEBUG'] = True + + if not EXPCONFIG['DEBUG']: + import monroe_exporter + else: + EXPCONFIG['verbosity'] = 3 + + + # Short hand variables and check so we have all variables we need + try: + guid=EXPCONFIG['guid'] + nodeid=EXPCONFIG['nodeid'] + verbosity = EXPCONFIG['verbosity'] + dataid = EXPCONFIG['dataid'] + dataversion = EXPCONFIG['dataversion'] + band=EXPCONFIG['band'] + resultdir=EXPCONFIG['resultdir'] + export_interval=EXPCONFIG['export_interval'] + device_path=EXPCONFIG['device'] + apn=EXPCONFIG['apn'] + interval = float(EXPCONFIG['interval']/1000.0) + server = EXPCONFIG['server'] + count = EXPCONFIG['count'] + pktsize = EXPCONFIG['size'] + dns_servers = EXPCONFIG['dns_servers'] + iptype = EXPCONFIG['type'] + except Exception as e: + print ("Missing expconfig variable {}".format(e)) + raise e + + if verbosity > 2: + print (EXPCONFIG) + + if not EXPCONFIG['DEBUG']: + monroe_exporter.initalize(export_interval, resultdir) + + #Load the helper functions + if verbosity > 1: + print("Loading the board: {}".format(device_path)) + pyb = pyboard.Pyboard(device=device_path) + if verbosity > 1: + print("Initalizing the board: {}".format(device_path)) + pyb.enter_raw_repl() + if pyb.execfile(filename="fipy-helper-functions.py") != b'': + print("Failed to load pycom helper functions") + raise Exception("Failed to load pycom helper functions") + + if verbosity > 1: + print("Getting ICCID: ", end = '', flush=True) + iccid = pyexec('get_iccid()', pyb) + if verbosity > 1: + print("{}".format(iccid)) + + assert(iccid) + + #Setup connections + if verbosity > 1: + print("Setting up dns {}: ".format(dns_servers), end = '', flush=True) + res = pyexec('set_dns({})'.format(dns_servers), pyb) + if verbosity > 1: + print("{}".format(res)) + + if verbosity > 1: + print('Setting up nbiot (apn="{}",band={}, type={}): '.format(apn, band, iptype), end = '', flush=True) + res = pyexec('set_nbiot(apn="{}",band={}, type={})'.format(apn, band, iptype), pyb) + if verbosity > 1: + print("{}".format(res)) + + # TODO: Check if lte is setup ok .... + cmd = 'ping(host="{}", count=1, size={})'.format(server, pktsize) + + seq = 0 + if verbosity > 1: + print("Starting experiment with {} runs: {}".format(count, cmd)) + while ("Connected" == pyexec('get_connection_status()', pyb) and + (seq < count or count < 1)): + if verbosity > 2: + print('Executing {}: '.format(cmd), end = '', flush=True) + try: + res = "" + res = pyexec(cmd, pyb) + exp_result = json.loads(res) + except Exception as e: + if verbosity > 2: + print('---> ', end = '', flush=True) + print("Failed execution/parsing of {} (got {})".format(cmd, res)) + raise e + + if exp_result and exp_result.get("n_recv",0) > 0: + msg = { + 'Bytes': int(exp_result['bytes']), + 'Host': server, + 'Rtt': float(exp_result["rtts"][0]), # We only have one value since count = 1 + 'SequenceNumber': int(seq), + 'Timestamp': time.time(), + "Guid": guid, + "DataId": dataid, + "DataVersion": dataversion, + "NodeId": nodeid, + "Iccid": iccid, + "Operator": apn + } + else: # We lost the interface or did not get a reply + msg = { + 'Host': server, + 'SequenceNumber': int(seq), + 'Timestamp': time.time(), + "Guid": guid, + "DataId": dataid, + "DataVersion": dataversion, + "NodeId": nodeid, + "Iccid": iccid, + "Operator": apn + } + + if verbosity > 2: + print (msg) + if not EXPCONFIG['DEBUG']: + # We have already initalized the exporter with the export dir + monroe_exporter.save_output(msg) + seq += 1 + time.sleep(interval) + # Cleanup + if verbosity > 1: + print ("We are not connected(anymore), cleaning up pycom board") + pyb.exit_raw_repl() diff --git a/experiments/pycom_ping/push.sh b/experiments/pycom_ping/push.sh new file mode 100755 index 0000000..85836e0 --- /dev/null +++ b/experiments/pycom_ping/push.sh @@ -0,0 +1,8 @@ +#/bin/sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +CONTAINER=${DIR##*/} + +CONTAINERTAG=monroe/${CONTAINER} + +docker login && docker tag ${CONTAINER} ${CONTAINERTAG} && docker push ${CONTAINERTAG} && echo "Finished uploading ${CONTAINERTAG}" diff --git a/experiments/pycom_ping/readme.md b/experiments/pycom_ping/readme.md new file mode 100644 index 0000000..a2710e9 --- /dev/null +++ b/experiments/pycom_ping/readme.md @@ -0,0 +1,78 @@ + +# Experiment +The experiments measure IP RTT by over NB IOT connections by continuously send ping packets to a configurable server (default 8.8.8.8, google public dns). + +The experiment will send 1 Echo Request (ICMP type 8) packet every 5 seconds to a server over the NBIOT interface until aborted or count is reached. +If count is set to < 1 the experiment will run forever. +RTT is measured as the time between the Echo request and the Echo reply +(ICMP type 0) is received from the server. + +The experiment is designed to run as a docker container and will only work with pycom LET enabled board (tested with fipy). + +The experiment is only as stable as the pycom device (ie fails quite often). + +The default values are (can be overridden by a /monroe/config): +``` +{ + "guid": "no.guid.in.config.file", # Should be overridden by scheduler + "nodeid": "fake.nodeid", + "server": "8.8.8.8", # ping target + "interval": 5000, # time in milliseconds between successive packets + "dataversion": 1, + "size":56, + "count":4, + "device": "/dev/pycom/board0", + "apn": "lpwa.telia.iot", + "type": "LTE.IP", #LTE.IP or LTE.IPV4V6 defualt LTE.IP + "band": None, # scans all bands + "dataid": "MONROE.EXP.NBPING", + "export_interval": 5.0, + "verbosity": 2, # 0 = "Mute", 1=error, 2=Information, 3=verbose + "resultdir": "/monroe/results/", + "dns_servers": ['8.8.8.8', '4.4.4.4'], + "DEBUG": False +} +``` +All debug/error information will be printed on stdout +depending on the "verbosity" variable. + +## Requirements + +These directories and files must exist and be read/writable by the user/process +running the container. +/monroe/config +"resultdir" (from /monroe/config see defaults above) + + +## Sample output +The experiment will produce a single line JSON object similar to these (pretty printed and added comments here for readability) +### Succesful reply +``` + { + 'Bytes': 56, + 'Host': '8.8.8.8', + 'Rtt': 539.817, + 'SequenceNumber': 0, + 'Timestamp': 1573218956.0079842, + 'Guid': '313.123213.123123.123123', + 'DataId': 'MONROE.EXP.NBPING', + 'DataVersion': 1, + 'NodeId': '9', + 'Iccid': '89450421190211492302', + 'Operator': 'lpwa.telia.iot' + } +``` +### No reply (lost interface or network issues) +``` + { + 'Host': '8.8.8.8', + 'SequenceNumber': 0, + 'Timestamp': 1573218956.0079842, + 'Guid': '313.123213.123123.123123', + 'DataId': 'MONROE.EXP.NBPING', + 'DataVersion': 1, + 'NodeId': '9', + 'Iccid': '89450421190211492302', + 'Operator': 'lpwa.telia.iot' + } +```