This page was generated from docs/devices/quick-xylo/xylo-audio-intro.ipynb. Interactive online version:

🐝🎵 Introduction to Xylo™Audio

Xylo™Audio is a platform for audio sensory processing, combining a low-power analog audio front-end with a low-power SNN inference core. Xylo™Audio is designed for sub-mW audio processing, in always-on applications.

This notebook gives you an overview of interfacing from Rockpool to the various cores of Xylo™Audio.

See also 🐝⚡️ Quick-start with Xylo SNN core and Using the analog frontend model.

Note: In this tutorial Xylo™Audio refers to Xylo™Audio 2 and Xylo™Audio 3. In case differences occur we will explicitly name the correct HDK.

# - Display images
from IPython.display import Image

Image('xylo-a2-block-level.png')

# - Display images
from IPython.display import Image

Image('xylo-a3-block-level.png')

Part I: Using the XyloAudio SNN Core

Interfacing with the XyloAudio SNN core is the same as for other Xylo family devices, making use of the Xylo deployment pipeline; syns61201.XyloSim and syns61201.XyloSamna, for XyloAudio 2; and syns65302.XyloSim and syns65302.XyloSamna, for XyloAudio 3.

See also 🐝⚡️ Quick-start with Xylo SNN core and 🐝 Overview of the Xylo™ family.

Image('XyloSamna.png', width=400)

Step 1: Build a network in rockpool and convert it to a hardware configuration

# - Import the computational modules and combinators required for the network
from rockpool.nn.modules import LIFTorch, LinearTorch
from rockpool.nn.combinators import Sequential, Residual
from rockpool.transform import quantize_methods as q
from rockpool import TSEvent, TSContinuous

# - Imports dependent on your HDK
# - XyloAudio 2
import rockpool.devices.xylo.syns61201 as xa2
from rockpool.devices.xylo.syns61201 import xa2_devkit_utils as xa2utils
# - XyloAudio 3
# import rockpool.devices.xylo.syns65302 as xa3
# from rockpool.devices.xylo.syns65302 import xa3_devkit_utils as xa3utils

import numpy as np

try:
    from rich import print
except:
    pass

import sys
!{sys.executable} -m pip install --quiet matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['figure.figsize'] = [12, 4]
plt.rcParams['figure.dpi'] = 300

# - Disable warnings
import warnings
warnings.filterwarnings('ignore')

from IPython import display

# - Define the size of the network layers
Nin = 2
Nhidden = 4
Nout = 2
dt = 1e-3

# - Define the network architecture using combinators and modules
net = Sequential(
    LinearTorch((Nin, Nhidden), has_bias = False),
    LIFTorch(Nhidden, dt = dt),

    Residual(
        LinearTorch((Nhidden, Nhidden), has_bias = False),
        LIFTorch(Nhidden, has_rec = True, threshold = 1., dt = dt),
    ),

    LinearTorch((Nhidden, Nout), has_bias = False),
    LIFTorch(Nout, dt = dt),
)
print(net)

# - Scale down recurrent weights for stability
# net[2][1].w_rec.data = net[2][1].w_rec / 10.

TorchSequential  with shape (2, 2) {
    LinearTorch '0_LinearTorch' with shape (2, 4)
    LIFTorch '1_LIFTorch' with shape (4, 4)
    TorchResidual '2_TorchResidual' with shape (4, 4) {
        LinearTorch '0_LinearTorch' with shape (4, 4)
        LIFTorch '1_LIFTorch' with shape (4, 4)
    }
    LinearTorch '3_LinearTorch' with shape (4, 2)
    LIFTorch '4_LIFTorch' with shape (2, 2)
}

# - Call the Xylo mapper on the extracted computational graph
# - For XyloAudio 2
spec = xa2.mapper(net.as_graph(),  weight_dtype='float', threshold_dtype='float', dash_dtype='float')
# - For XyloAudio 3
# spec = xa3.mapper(net.as_graph(),  weight_dtype='float', threshold_dtype='float', dash_dtype='float')

# - Quantize the specification
spec.update(q.global_quantize(**spec))

# # you can also try channel-wise quantization
# spec.update(q.channel_quantize(**spec))
# print(spec)

# - Use rockpool.devices.xylo.config_from_specification to convert it to a hardware configuration
# - For XyloAudio 2
config, is_valid, msg = xa2.config_from_specification(**spec)
# - For XyloAudio 3
# config, is_valid, msg = xa3.config_from_specification(**spec)

if not is_valid:
    print(msg)

# - Use rockpool.devices.xylo.find_xylo_hdks to connect to an HDK
from rockpool.devices.xylo import find_xylo_hdks
xylo_hdk_nodes, modules, versions = find_xylo_hdks()
print(versions)

hdk = None

for version, xylo in zip(versions, xylo_hdk_nodes):
    if version == "syns61201":
        hdk = xylo
    # - For XyloAudio 3
    # if version == "syns65302":
    #     hdk = xylo

if hdk is None:
    assert False, 'This tutorial requires a connected XyloAudio HDK to demonstrate.'

The connected Xylo HDK contains a Xylo Audio v2 (SYNS61201). Importing `rockpool.devices.xylo.syns61201`

['syns61201']

# - Use XyloSamna to deploy to the HDK
if hdk:
    # - For XyloAudio 2
    modSamna = xa2.XyloSamna(hdk, config, dt = dt)
    # - For XyloAudio 3
    # modSamna = xa3.XyloSamna(hdk, config, dt = dt)

print(modSamna)

XyloSamna  with shape (2, 8, 2)

# - Generate some Poisson input
T = 100
f = 0.4
input_spikes = np.random.rand(T, Nin) < f
TSEvent.from_raster(input_spikes, dt, name = 'Poisson input events').plot();

# - Evolve the network on the Xylo HDK
# - `reset_state` is only needed for XyloAudio 2
modSamna.reset_state()
out, _, r_d = modSamna(input_spikes, record = True)

# - Show the internal state variables recorded
print(r_d.keys())

dict_keys(['Vmem', 'Isyn', 'Isyn2', 'Spikes', 'Vmem_out', 'Isyn_out', 'times'])

# - Plot some internal state variables
plt.figure()
plt.imshow(r_d['Spikes'].T, aspect = 'auto', origin = 'lower')
plt.title('Hidden spikes')
plt.ylabel('Channel')

plt.figure()
TSContinuous(r_d['times'], r_d['Isyn'], name = 'Hidden synaptic currents').plot(stagger = 127)

plt.figure()
TSContinuous(r_d['times'], r_d['Vmem'], name = 'Hidden membrane potentials').plot(stagger = 127);

Step 3: Simulate the HDK using the XyloSim bit-precise simulator

# - Configure the simulator with the HW network config
# - For XyloAudio 2
modSim = xa2.XyloSim.from_config(config, dt=dt)
# - For XyloAudio 3
# modSim = xa3.XyloSim.from_config(config, dt=dt)

print(modSim)

XyloSim  with shape (16, 1000, 8)

# - Evolve the input over the network, in simulation
out, _, r_d = modSim(input_spikes, record = True)

# - Show the internal state variables recorded
print(r_d.keys())

dict_keys(['Vmem', 'Isyn', 'Isyn2', 'Spikes', 'Vmem_out', 'Isyn_out'])

# - Plot some internal state variables
plt.figure()
plt.imshow(r_d['Spikes'].T, aspect = 'auto', origin = 'lower')
plt.title('Hidden spikes')
plt.ylabel('Channel')

plt.figure()
TSContinuous.from_clocked(r_d['Isyn'], dt, name = 'Hidden synaptic currents').plot(stagger = 127);

plt.figure()
TSContinuous.from_clocked(r_d['Vmem'], dt, name = 'Hidden membrane potentials').plot(stagger = 127);

Part II: Using the XyloAudio audio front-end interface

Note: Audio front-end is only available in Xylo™Audio 2. This step does not applied to Xylo™Audio 3.

The AFE (Audio Front-End) is used to preprocess single-channel audio signals and convert them into spikes. Here the audio signal is input to the AFE by a microphone mounted on the hardware dev kit, or an external differential analog audio signal. The AFE has 16 output channels, and you can adjust its parameters via hyperparameters in the AFESamna class.

See also the introductory notebook Using the analog frontend model .

AFESamna allows you to access the audio front-end on the dev kit, and record encoded audio either from the on-board microphone or analog audio injected to the dev kit.

AFESamna also allows a custom config input which is without auto-calibration. If you do not provide a custom config, we highly suggest you set auto_calibrate = True on instantiation, which helps to mitigate the effects of background and mechanical noise.

Image('AFESamna.png', width=400)

# - Find and connect to a XyloAudio HDK
from rockpool.devices.xylo import find_xylo_hdks
xylo_hdk_nodes, modules, versions = find_xylo_hdks()
print(versions)

hdk = None

for version, xylo in zip(versions, xylo_hdk_nodes):
    if version == "syns61201":
        hdk = xylo

if hdk is None:
    assert False, 'This tutorial requires a connected XyloAudio HDK to demonstrate.'

The connected Xylo HDK contains a Xylo Audio v2 (SYNS61201). Importing `rockpool.devices.xylo.syns61201`

[<samna.xyloA2TestBoard.XyloA2TestBoard object at 0x299255bf0>]

# - Set the time resolution and duration to record encoded audio
dt = 10e-3
timesteps = 1000

# - Create an AFESamna module, which wraps the AFE on the Xylo A2 HDK
#   (stay quiet while this cell is executing)
mod = xa2.AFESamna(hdk, None, dt=dt, auto_calibrate=True, amplify_level='low', hibernation_mode=False)
print(mod)

Configuring AFE...
Configured AFE

AFESamna  with shape (0, 16)

# - Evolve the module to record encoded real-time audio as events
spikes_ts, _, _ = mod(np.zeros([0, timesteps, 0]))

# - Plot some encoded audio events recorded from the AFE
plt.imshow(spikes_ts.T, aspect='auto', interpolation='none')
plt.title('#Spikes in AFE output channels')
plt.xlabel('Time')
plt.ylabel('Channel')
plt.show()

You can record encoded audio using the AFESamna class, and feed these outputs to the Xylo core for testing using XyloSamna. You can also combine the two cores for inference, as shown in the following section.

Part III: Deploying the AFE and SNN cores in free-running inference mode

Once you have a complete chip HW specification, you can deploy it to the chip in real-time inference mode using the classes syns61201.XyloMonitor or syns65302.XyloMonitor for XyloAudio 2 and XyloAudio 3, respectively. In XyloAudio 2, this mode uses the AFE core to preprocess audio signals in real-time and then sends encoded audio to the SNN core for inference. In XyloAudio 3, this mode uses the digital microphone audio signal in real-time and then sends encoded audio to the SNN core for inference. In this mode you only read the output events from the SNN core, without providing input.

Image('XyloMonitor.png', width=400)

# - Find and connect to a XyloAudio HDK
xylo_hdk_nodes, modules, versions = find_xylo_hdks()
print(xylo_hdk_nodes)

hdk = None

for version, xylo in zip(versions, xylo_hdk_nodes):
    if version == "syns61201":
        hdk = xylo
    # - For XyloAudio 3
    # if version == "syns65302":
    #     hdk = xylo

if hdk is None:
    assert False, 'This tutorial requires a connected XyloAudio HDK to demonstrate.'

The connected Xylo HDK contains a Xylo Audio v2 (SYNS61201). Importing `rockpool.devices.xylo.syns61201`

[<samna.xyloA2TestBoard.XyloA2TestBoard object at 0x299255bf0>]

# - Use XyloMonitor to deploy to the HDK
# - For XyloAudio 2 you need to wait 45s until the AFE auto-calibration is done

output_mode = "Vmem"
amplify_level = "low"
hibernation = False
DN = False
T = 10

# - For XyloAudio 2
modMonitor = xa2.XyloMonitor(hdk, config, dt=dt, output_mode=output_mode, amplify_level=amplify_level, hibernation_mode=hibernation, divisive_norm=DN)
# - For XyloAudio 3
# - XyloAudio 3 does not have the amplify_level parameter and does not do AFE auto-calibration
# modMonitor = xa3.XyloMonitor(hdk, config, dt = dt, output_mode=output_mode, hibernation_mode=hibernation, dn_active=DN)

Configuring AFE...
Configured AFE

# - A resultList stack to store the results

class ResultList(object):
    def __init__(self, max_len=100):
        self._list = []
        self.max_len = max_len

    def reset(self):
        self._list = []

    def append(self, num):
        if len(self._list) < self.max_len:
            self._list.append(num)
        else:
            self._list[: self.max_len - 1] = self._list[1:]
            self._list[self.max_len - 1] = num

    def is_full(self):
        if len(self._list) == self.max_len:
            return True
        else:
            return False

    def counts(self, features=[]):
        count = 0
        for _ in self._list:
            if _ in features:
                count += 1
        return count

    def __len__(self):
        return len(self._list)

    def print_result(self):
        return self._list

# - Draw a real time image for output channels
lines = [ResultList(max_len=10) for _ in range(Nout)]
time_base = ResultList(max_len=10)
tt = 0
t_inference = 10.


from time import time

t_start = time()

while (time() - t_start) < t_inference:
    # - Perform inference on the Xylo A2 HDK
    output, _, _ = modMonitor(input_data=np.zeros((T, Nin)))
    if output is not None:
        output = np.max(output, axis=0)
        for i in range(Nout):
            lines[i].append(output[i])

        time_base.append(tt)
        tt += 0.1
        ax_time = time_base.print_result()

        for i in range(Nout):
            plt.plot(ax_time, lines[i].print_result(), label=f"class{i}")

        plt.xlabel('time')
        plt.ylabel('Vmem')
        plt.legend()
        plt.pause(0.1)
        display.clear_output(wait=True)

Part IV: Measuring power on the XyloAudio HDK

syns61201.XyloSamna and syns65302.XyloSamna (respectively for XyloAudio 2 and XyloAudio 3) provide an interface to real-time power measurements on the XyloAudio HDK. Current on several power nets on the chip can be sampled asynchronously, while the device is in operation.

The evolve() method provides a power measurement interface while the device is in inference mode, and the XyloSamna module can be used also to measure idle power.

# - Find and connect to a XyloAudio HDK
xylo_hdk_nodes, modules, versions = find_xylo_hdks()
print(xylo_hdk_nodes)

hdk = None

for version, xylo in zip(versions, xylo_hdk_nodes):
    if version == "syns61201":
        hdk = xylo
    # - For XyloAudio 3
    # if version == "syns65302":
    #     hdk = xylo

if hdk is None:
    assert False, 'This tutorial requires a connected XyloAudio HDK to demonstrate.'

The connected Xylo HDK contains a Xylo Audio v2 (SYNS61201). Importing `rockpool.devices.xylo.syns61201`

[<samna.xyloA2TestBoard.XyloA2TestBoard object at 0x299255bf0>]

On instantiation, XyloSamna can specify a power sampling frequency. By default, power is measured at 5 Hz.

# - Set a low clock frequency for the XyloAudio 2 device
# - This step is not necessary for XyloAudio 3
xa2utils.set_xylo_core_clock_freq(hdk, 6.25)

# - Use XyloSamna to deploy to the HDK
# - For XyloAudio 2
modSamna = xa2.XyloSamna(hdk, config, dt = 10e-3, power_frequency=20.)
# - For XyloAudio 3
# modSamna = xa3.XyloSamna(hdk, config, dt = 10e-3, power_frequency=20.)
print(modSamna)

XyloSamna  with shape (2, 8, 2)

# - Generate some Poisson input
T = 1000
f = 0.4
input_spikes = np.random.rand(T, Nin) < f

# - Evolve some input on the SNN core, and record power during inference
out, _, record_dict = modSamna(input_spikes, record_power = True)

print(record_dict)

{
    'io_power': array([0.00010319, 0.00010395, 0.000103  , 0.00010395, 0.00010529,
       0.00010319]),
    'logic_afe_power': array([1.07421875e-05, 1.22528076e-05, 9.56726074e-06, 1.09100342e-05,
       1.07421875e-05, 9.90295410e-06]),
    'io_afe_power': array([8.96453857e-06, 6.29425049e-06, 8.20159912e-06, 6.48498535e-06,
       9.72747803e-06, 7.43865967e-06]),
    'logic_power': array([0.00027166, 0.00027309, 0.00027233, 0.00026931, 0.00027107,
       0.00026998])
}

For XyloAudio 2 power is sampled on four nets, and is returned in Watts.

‘io_power’ is the total I/O power of the device. ‘snn_core_power’ is the power consumption of the digital SNN core and control logic. ‘afe_core_power’ is the power of the analog audio front-end core. ‘afe_ldo_power’ is the power consumption of the internal low-drop-out voltage supply used by the AFE.

For XyloAudio 3 power is sampled on three nets, and is returned in Watts.

‘io_power’ is the total I/O power of the device. ‘analog_power’ is the power of the analog audio front-end core. ‘digital_power’ is the power consumption of the digital SNN core and control logic.

# - Measure idle power (no evolution)
from time import sleep
import samna

modSamna._power_buf.get_events()
sleep(5.)
power = modSamna._power_buf.get_events()

power_idle = ([], [], [], [])

for p in power:
    power_idle[p.channel].append(p.value)

idle_power_per_channel = np.mean(np.stack(power_idle), axis = 1)

# - For XyloAudio 2
channels = samna.xyloA2TestBoard.MeasurementChannels
io_power = idle_power_per_channel[channels.Io]
afe_core_power = idle_power_per_channel[channels.LogicAfe]
afe_ldo_power = idle_power_per_channel[channels.IoAfe]
snn_core_power = idle_power_per_channel[channels.Logic]

# - For XyloAudio 3
# io_power = idle_power_per_channel[0]
# analog_power = idle_power_per_channel[1]
# digital_power = idle_power_per_channel[2]


print(f'XyloAudio 2\nAll IO:\t\t{io_power * 1e6:.1f} µW\nAFE core:\t{afe_core_power * 1e6:.1f} µW\nInternal LDO:\t{afe_ldo_power * 1e6:.1f} µW\nSNN core logic:\t{snn_core_power*1e6:.1f} µW')
print(f'XyloAudio 3\nAll IO:\t\t{io_power * 1e6:.1f} µW\nAFE core:\t{afe_core_power * 1e6:.1f} µW\nSNN core logic:\t{snn_core_power*1e6:.1f} µW')

All IO:         99.84207153320312µW
AFE analog:     10.385513305664062µW
Internal LDO:   7.274627685546874µW
SNN core logic: 250.0865173339844µW

Hints on reducing power consumption

Several points may be useful in reducing active power on XyloAudio.

• Reducing the core clock frequency. The dev-kit utility function devices.xylo.syns61201.xa2_devkit_utils.set_xylo_core_clock_freq() can be used to set a lower core clock frequency than the default of 50 Mhz, lowering power consumption proportionally. Depending on your network size and complexity you can most likely reduce the clock frequency and still guarantee real-time operation.

• Reducing network activity and network size. The SNN core within Xylo only works as hard as it needs to. Once all neuron states have been updated, the core performs no operations until the next network time-step. This conserves energy.

help(xa2utils.set_xylo_core_clock_freq)

Help on function set_xylo_core_clock_freq in module rockpool.devices.xylo.syns61201.xa2_devkit_utils:

set_xylo_core_clock_freq(device: Any, desired_freq_MHz: float) -> float
    Set the inference core clock frequency used by Xylo

    Args:
        device (XyloA2HDK): A Xylo2 device to configure
        desired_freq_MHz (float): The desired Xylo2 core clock frequency in MHz

    Returns:
        float: The obtained Xylo2 core clock frequency in MHz

Estimate the required master clock frequency for real-time operation

Rockpool includes a model of the cycles required to operate the Xylo SNN inference core, in cycles_model(). Xylo is an efficient device, which skips computation whenever possible. As a result, the computational requirements and power consumption vary dynamically with network activity, and with network configuration.

By default cycles_model() estimates a worst-case scenario, which can inform the master clock frequency that guarantees real-time operation for your network. It can also take into account real network activity to estimate a usual computational load.

The function est_clock_freq() will return the master clock frequency that guarantees real-time operation, for a given Xylo configuration.

from rockpool.devices.xylo.syns61201 import cycles_model, est_clock_freq

help(cycles_model)

Help on function cycles_model in module rockpool.devices.xylo.syns61201.power_cycles_model:

cycles_model(config: samna.xyloCore2.configuration.XyloConfiguration, input_sp: Union[float, numpy.ndarray, torch.Tensor, array] = 1.0, hidden_sp: Union[float, numpy.ndarray, torch.Tensor, array] = 1.0, output_sp: Union[float, numpy.ndarray, torch.Tensor, array] = 1.0) -> Tuple[float, float]
    Calculate the average number of cycles required for a given network architecture

    This function contains a model which estimates the number of master clock cycles required for the Xylo SNN SYNS61202 inference core to compute one time-step for a given chip configuration in ``config``. Use :py:func:`.devices.xylo.syns61201.config_from_specification`` to obtain a chip configuration, along with :py:meth:`.Module.as_graph()`` and :py:func:`.devices.xylo.syns61201.mapper`, as described in the deployment tutorials for Xylo.

    By default the model provides a "worst-case" estimation, assuming that every neuron and every input channel fire on each time-step. If desired, real input rasters and real hidden and output spike rasters can be provided for analysis. Alternative spiking probabilities can also be provided as floats ``0..1``.

    Note that when estimating spiking probablility, only boolean values are relevant --- either a spike or no spike per time step per channel. Multiple events per bin cost the same as a single event.

    Args:
        config (XyloA2Config): A Xylo Audio 2 configuration for which to calculate the cycle requirements
        input_sp (FloatVector): Either a floating-point number 0..1, specifying the average input firing rate, or an actual input spike raster to use in evaluation. Default: `1.0`; estimate a worst-case scenario
        hidden_sp (FloatVector): Either a floating-point number 0..1, specifying the average hidden neuron firing rate, or an actual hidden spike raster to use in evaluation. Default: `1.0`; estimate a worst-case scenario
        output_sp (FloatVector): Either a floating-point number 0..1, specifying the average output neuron firing rate, or an actual output spike raster to use in evaluation. Default: `1.0`; estimate a worst-case scenario

    Returns:
        Tuple[float, float]: (clk_cyles_reqd, additional_isyn2_ops)
        clk_cyles_reqd (float): The average number of master clock cycles required for this configuration, for the Xylo SNN core to compute one network `dt`
        additional_isyn2_ops (float): The average additional number of operations required when Isyn2 is enabled. Will be zero if Isyn2 is disabled. Note: These operations occur in parallel to Isyn1, and do not require additional master clock cycles.

help(est_clock_freq)

Help on function est_clock_freq in module rockpool.devices.xylo.syns61201.power_cycles_model:

est_clock_freq(config: samna.xyloCore2.configuration.XyloConfiguration, dt: float, margin: float = 0.2)
    Estimate the required master clock frequency, to run a network in real-time

    This function will perform a worst-case analysis, assuming that every input channel, every hidden neuron and every output neuron fire an event on each `dt`. An additional margin is included (Default: 20%), to guarantee that the model will run in real time at the suggested master clock frequency.

    Args:
        config (XyloA2Config):  A Xylo Audio 2 configuration for which to estimate the required clock frequency
        dt (float): The required network `dt`, in seconds
        margin (float): The additional overhead safety margin to add to the estimation, as a fraction. Default: `0.2` (20%)

print(f"This network requires {cycles_model(config)} master clock cycles per network time-step.")
print(f"This network requires a master clock of {est_clock_freq(config, dt) / 1e6:.2f} MHz for real-time operation.")