message: Added resampling and tests. Small edits to plotting like documentation.

diff:
@@ -15,6 +15,7 @@ import audioread
import json
import warnings
import copy
+import resampy
import matplotlib.pyplot as plt
import separation.masks.mask_base
@@ -77,7 +78,7 @@ class AudioSignal(object):
self.path_to_input_file = path_to_input_file
self._audio_data = None
self._stft_data = None
- self.sample_rate = sample_rate
+ self._sample_rate = sample_rate
self._active_start = None
self._active_end = None
@@ -106,105 +107,6 @@ class AudioSignal(object):
def __str__(self):
return self.__class__.__name__
- ##################################################
- # Plotting
- ##################################################
- #
-
- _NAME_STEM = 'audio_signal'
-
- def plot_time_domain(self, channel=None, x_label_time=True, save=False, name=None, output_path=None):
- """
- Plots a graph of the time domain audio signal
- Parameters:
- x_label (str): Label the x axis with time or samples
- save (bool): Save a png of the plot to the current folder
- name (str): The name of the audio signal. Applies to title and filename
- output_path (str): The output path of where the plot is saved if save is True. Should include '/' at the end
-
- """
-
- if self.audio_data is None:
- raise ValueError('Cannot plot with no audio data!')
-
- if channel > self.num_channels-1:
- raise ValueError('Channel selected does not exist!')
-
- # Mono or single specific channel selected for plotting
- if self.num_channels == 1 or channel is not None:
- plot_channels = channel if channel else self.num_channels-1
- if x_label_time is True:
- plt.plot(self.time_vector, self.audio_data[plot_channels])
- plt.xlim(self.time_vector[0], self.time_vector[-1])
- else:
- plt.plot(self.audio_data[plot_channels])
- plt.xlim(0, self.signal_length)
- channel_num_plot = 'Channel {}'.format(plot_channels)
- plt.ylabel(channel_num_plot)
-
- # Stereo signal plotting
- elif self.num_channels == 2 and channel is None:
- top_plot = abs(self.audio_data[0])
- bottom_plot = -abs(self.audio_data[1])
- if x_label_time is True:
- plt.plot(self.time_vector, top_plot)
- plt.plot(self.time_vector, bottom_plot, 'C0')
- plt.xlim(self.time_vector[0], self.time_vector[-1])
- else:
- plt.plot(top_plot)
- plt.plot(bottom_plot, 'C0')
- plt.xlim(0, self.signal_length)
-
- # Plotting more than 2 channels each on their own plots in a stack
- elif self.num_channels > 2 and channel is None:
- for i in range(self.num_channels):
- plt.subplot(self.num_channels+1, 1, i+1)
- if x_label_time is True:
- plt.plot(self.time_vector, self.audio_data[i])
- plt.xlim(self.time_vector[0], self.time_vector[-1])
- else:
- plt.plot(self.audio_data[i])
- plt.xlim(0, self.signal_length)
- channel_num_plot = 'Channel {}'.format(i)
- plt.ylabel(channel_num_plot)
-
- if name is None:
- name = self.file_name if self.file_name is not None else self._NAME_STEM
- else:
- name = os.path.splitext(name)[0]
-
- plt.suptitle(name)
- name = name if self._check_if_valid_img_type(name) else name + '.png'
-
- if save:
- output_name_path = output_path if output_path else ''
- output_name_path += name
- plt.savefig(output_name_path)
-
- def plot_spectrogram(self, file_name=None, ch=None):
- # TODO: use self.stft_data if not None
- # TODO: flatten to mono be default
- # TODO: make other parameters adjustable
- if file_name is None:
- name = self.file_name if self.file_name is not None else self._NAME_STEM + '_spectrogram'
- else:
- name = os.path.splitext(file_name)[0]
-
- name = name if self._check_if_valid_img_type(name) else name + '.png'
-
- if ch is None:
- spectral_utils.plot_stft(self.to_mono(), name, sample_rate=self.sample_rate)
- else:
- spectral_utils.plot_stft(self.get_channel(ch), name, sample_rate=self.sample_rate)
-
- @staticmethod
- def _check_if_valid_img_type(name):
- import matplotlib.pyplot as plt
- fig = plt.figure()
- result = any([name[len(k):] == k for k in fig.canvas.get_supported_filetypes().keys()])
- plt.close()
- return result
-
##################################################
# Properties
##################################################
@@ -338,6 +240,11 @@ class AudioSignal(object):
return os.path.split(self.path_to_input_file)[1]
return None
+ @property
+ def sample_rate(self):
+ return self._sample_rate
+
+
@property
def time_vector(self):
""" (:obj:`np.ndarray`): A 1D np array with timestamps (in seconds) for each sample in the time domain.
@@ -483,7 +390,7 @@ class AudioSignal(object):
# I/O
##################################################
- def load_audio_from_file(self, input_file_path, offset=0, duration=None):
+ def load_audio_from_file(self, input_file_path, offset=0, duration=None, new_sample_rate=None):
# type: (str, float, float) -> None
"""Loads an audio signal from a file
@@ -518,6 +425,10 @@ class AudioSignal(object):
self.audio_data = audio_input
+ if new_sample_rate is not None and new_sample_rate != self._sample_rate:
+ warnings.warn("Input sample rate is different than the sample rate read from the file! Resampling...", UserWarning)
+ self.resample(new_sample_rate)
+
except Exception as e:
if isinstance(e, ValueError): # This is the error we just raise, re-raise it
raise e
@@ -527,7 +438,7 @@ class AudioSignal(object):
self.path_to_input_file = input_file_path
self.set_active_region_to_default()
- def load_audio_from_array(self, signal, sample_rate=constants.DEFAULT_SAMPLE_RATE):
+ def load_audio_from_array(self, signal, sample_rate=constants.DEFAULT_SAMPLE_RATE, new_sample_rate=None):
"""Loads an audio signal from a numpy array.
Notes:
@@ -551,6 +462,11 @@ class AudioSignal(object):
self.audio_data = signal
self._sample_rate = sample_rate
+
+ if new_sample_rate is not None and new_sample_rate != self._sample_rate:
+ warnings.warn("Input sample rate is different than the default sample rate! Resampling...", UserWarning)
+ self.resample(new_sample_rate)
+
self.set_active_region_to_default()
def write_audio_to_file(self, output_file_path, sample_rate=None, verbose=False):
@@ -801,6 +717,106 @@ class AudioSignal(object):
masked_stft = self.stft_data * mask.mask
return self.make_copy_with_stft_data(masked_stft, verbose=False)
+ ##################################################
+ # Plotting
+ ##################################################
+ #
+
+ _NAME_STEM = 'audio_signal'
+
+ def plot_time_domain(self, channel=None, x_label_time=True, save=False, name=None, output_path=None):
+ """
+ Plots a graph of the time domain audio signal
+ Parameters:
+ channel (int): The index of the single channel to be plotted
+ x_label_time (True): Label the x axis with time (True) or samples (False)
+ save (bool): Save a png of the plot to the current folder
+ name (str): The name of the audio signal. Applies to title and filename
+ output_path (str): The output path of where the plot is saved if save is True. Should include '/' at the end
+
+ """
+
+ if self.audio_data is None:
+ raise ValueError('Cannot plot with no audio data!')
+
+ if channel > self.num_channels - 1:
+ raise ValueError('Channel selected does not exist!')
+
+ # Mono or single specific channel selected for plotting
+ if self.num_channels == 1 or channel is not None:
+ plot_channels = channel if channel else self.num_channels - 1
+ if x_label_time is True:
+ plt.plot(self.time_vector, self.audio_data[plot_channels])
+ plt.xlim(self.time_vector[0], self.time_vector[-1])
+ else:
+ plt.plot(self.audio_data[plot_channels])
+ plt.xlim(0, self.signal_length)
+ channel_num_plot = 'Channel {}'.format(plot_channels)
+ plt.ylabel(channel_num_plot)
+
+ # Stereo signal plotting
+ elif self.num_channels == 2 and channel is None:
+ top_plot = abs(self.audio_data[0])
+ bottom_plot = -abs(self.audio_data[1])
+ if x_label_time is True:
+ plt.plot(self.time_vector, top_plot)
+ plt.plot(self.time_vector, bottom_plot, 'C0')
+ plt.xlim(self.time_vector[0], self.time_vector[-1])
+ else:
+ plt.plot(top_plot)
+ plt.plot(bottom_plot, 'C0')
+ plt.xlim(0, self.signal_length)
+
+ # Plotting more than 2 channels each on their own plots in a stack
+ elif self.num_channels > 2 and channel is None:
+ for i in range(self.num_channels):
+ plt.subplot(self.num_channels + 1, 1, i + 1)
+ if x_label_time is True:
+ plt.plot(self.time_vector, self.audio_data[i])
+ plt.xlim(self.time_vector[0], self.time_vector[-1])
+ else:
+ plt.plot(self.audio_data[i])
+ plt.xlim(0, self.signal_length)
+ channel_num_plot = 'Channel {}'.format(i)
+ plt.ylabel(channel_num_plot)
+
+ if name is None:
+ name = self.file_name if self.file_name is not None else self._NAME_STEM
+ else:
+ name = os.path.splitext(name)[0]
+
+ plt.suptitle(name)
+ name = name if self._check_if_valid_img_type(name) else name + '.png'
+
+ if save:
+ output_name_path = output_path if output_path else ''
+ output_name_path += name
+ plt.savefig(output_name_path)
+
+ def plot_spectrogram(self, file_name=None, ch=None):
+ # TODO: use self.stft_data if not None
+ # TODO: flatten to mono be default
+ # TODO: make other parameters adjustable
+ if file_name is None:
+ name = self.file_name if self.file_name is not None else self._NAME_STEM + '_spectrogram'
+ else:
+ name = os.path.splitext(file_name)[0]
+
+ name = name if self._check_if_valid_img_type(name) else name + '.png'
+
+ if ch is None:
+ spectral_utils.plot_stft(self.to_mono(), name, sample_rate=self.sample_rate)
+ else:
+ spectral_utils.plot_stft(self.get_channel(ch), name, sample_rate=self.sample_rate)
+
+ @staticmethod
+ def _check_if_valid_img_type(name):
+ import matplotlib.pyplot as plt
+ fig = plt.figure()
+ result = any([name[len(k):] == k for k in fig.canvas.get_supported_filetypes().keys()])
+ plt.close()
+ return result
+
##################################################
# Utilities
##################################################
@@ -1150,6 +1166,20 @@ class AudioSignal(object):
self.audio_data = audio_data
return self
+ def resample(self, new_sample_rate):
+ """
+ Resample an audio signal using resample
+ Args:
+ new_sample_rate: (int) The new sample rate to apply to the audio signal
+ overwrite: (bool) should overwrite audio_data in self
+
+ Returns:
+
+ """
+ resampled_signal = resampy.resample(self.audio_data, self.sample_rate, new_sample_rate)
+ self.audio_data = resampled_signal
+ self._sample_rate = new_sample_rate
+
##################################################
# Channel Utilities
##################################################

diff:
@@ -180,6 +180,30 @@ class AudioSignalUnitTests(unittest.TestCase):
freq = 30
sine_wave = np.sin(np.linspace(0, freq * 2 * np.pi, length))
+ def test_resample(self):
+ a = nussl.AudioSignal(self.input_path1)
+ b = nussl.AudioSignal(self.input_path1)
+ b.resample(a.sample_rate/2)
+ assert (b.sample_rate == a.sample_rate/2)
+
+ def test_resample_on_load_from_file(self):
+ a = nussl.AudioSignal(self.input_path1)
+ a.resample(48000)
+ b = nussl.AudioSignal()
+ b.load_audio_from_file(self.input_path1, new_sample_rate=48000)
+ assert (a.sample_rate== b.sample_rate)
+ assert (np.allclose(a.audio_data, b.audio_data))
+
+ def test_resample_on_load_from_array(self):
+ sr, data = wav.read(self.input_path1)
+ a = nussl.AudioSignal()
+ a.load_audio_from_array(data, sample_rate=sr)
+ a.resample(48000)
+ b = nussl.AudioSignal()
+ b.load_audio_from_array(data, sample_rate=sr, new_sample_rate=48000)
+ assert (a.sample_rate == b.sample_rate)
+ assert (np.allclose(a.audio_data, b.audio_data))
+
def test_plot_time_domain_stereo(self):
# Stereo signal that should plot both channels on same plot
a = nussl.AudioSignal(self.input_path3)


message: Added resampling and tests.

change_type: MODIFY

diff:
@@ -15,7 +15,6 @@ import audioread
import json
import warnings
import copy
-import resampy
import matplotlib.pyplot as plt
import separation.masks.mask_base
@@ -1176,7 +1175,7 @@ class AudioSignal(object):
Returns:
"""
- resampled_signal = resampy.resample(self.audio_data, self.sample_rate, new_sample_rate)
+ resampled_signal = librosa.resample(self.audio_data, self.sample_rate, new_sample_rate)
self.audio_data = resampled_signal
self._sample_rate = new_sample_rate

license: MIT License

repo: nussl/nussl

original_message: Added resampling and tests.


message: Added x-axis sharing for plotting and multichannel resampling and tests.

diff:
@@ -72,12 +72,12 @@ class AudioSignal(object):
"""
def __init__(self, path_to_input_file=None, audio_data_array=None, stft=None,
- sample_rate=constants.DEFAULT_SAMPLE_RATE, stft_params=None, offset=0, duration=None):
+ sample_rate=None, stft_params=None, offset=0, duration=None):
self.path_to_input_file = path_to_input_file
self._audio_data = None
self._stft_data = None
- self._sample_rate = sample_rate
+ self._sample_rate = None
self._active_start = None
self._active_end = None
@@ -92,9 +92,11 @@ class AudioSignal(object):
raise ValueError('Can only initialize AudioSignal object with one of [path, audio, stft]!')
if path_to_input_file is not None:
- self.load_audio_from_file(self.path_to_input_file, offset, duration)
+ self.load_audio_from_file(self.path_to_input_file, offset, duration, sample_rate)
elif audio_data_array is not None:
self.load_audio_from_array(audio_data_array, sample_rate)
+ else:
+ self._sample_rate = constants.DEFAULT_SAMPLE_RATE
# stft data
if stft is not None:
@@ -425,7 +427,8 @@ class AudioSignal(object):
self.audio_data = audio_input
if new_sample_rate is not None and new_sample_rate != self._sample_rate:
- warnings.warn("Input sample rate is different than the sample rate read from the file! Resampling...", UserWarning)
+ warnings.warn("Input sample rate is different than the sample rate read from the file! Resampling...",
+ UserWarning)
self.resample(new_sample_rate)
except Exception as e:
@@ -437,7 +440,7 @@ class AudioSignal(object):
self.path_to_input_file = input_file_path
self.set_active_region_to_default()
- def load_audio_from_array(self, signal, sample_rate=constants.DEFAULT_SAMPLE_RATE, new_sample_rate=None):
+ def load_audio_from_array(self, signal, sample_rate=constants.DEFAULT_SAMPLE_RATE):
"""Loads an audio signal from a numpy array.
Notes:
@@ -460,11 +463,7 @@ class AudioSignal(object):
signal = signal.astype('float') / (np.iinfo(np.dtype('int16')).max + 1.0)
self.audio_data = signal
- self._sample_rate = sample_rate
-
- if new_sample_rate is not None and new_sample_rate != self._sample_rate:
- warnings.warn("Input sample rate is different than the default sample rate! Resampling...", UserWarning)
- self.resample(new_sample_rate)
+ self._sample_rate = sample_rate if sample_rate is not None else constants.DEFAULT_SAMPLE_RATE
self.set_active_region_to_default()
@@ -723,15 +722,14 @@ class AudioSignal(object):
_NAME_STEM = 'audio_signal'
- def plot_time_domain(self, channel=None, x_label_time=True, save=False, name=None, output_path=None):
+ def plot_time_domain(self, channel=None, x_label_time=True, title=None, file_path_name=None):
"""
Plots a graph of the time domain audio signal
Parameters:
channel (int): The index of the single channel to be plotted
x_label_time (True): Label the x axis with time (True) or samples (False)
- save (bool): Save a png of the plot to the current folder
- name (str): The name of the audio signal. Applies to title and filename
- output_path (str): The output path of where the plot is saved if save is True. Should include '/' at the end
+ title (str): The title of the audio signal plot
+ file_path_name (str): The output path of where the plot is saved, including the file name
"""
@@ -768,29 +766,28 @@ class AudioSignal(object):
# Plotting more than 2 channels each on their own plots in a stack
elif self.num_channels > 2 and channel is None:
+ f, axarr = plt.subplots(self.num_channels, sharex=True)
for i in range(self.num_channels):
- plt.subplot(self.num_channels + 1, 1, i + 1)
if x_label_time is True:
- plt.plot(self.time_vector, self.audio_data[i])
- plt.xlim(self.time_vector[0], self.time_vector[-1])
+ axarr[i].plot(self.time_vector, self.audio_data[i])
+ axarr[i].set_xlim(self.time_vector[0], self.time_vector[-1])
else:
- plt.plot(self.audio_data[i])
- plt.xlim(0, self.signal_length)
- channel_num_plot = 'Channel {}'.format(i)
- plt.ylabel(channel_num_plot)
+ axarr[i].plot(self.audio_data[i], sharex=True)
+ axarr[i].set_xlim(0, self.signal_length)
+ channel_num_plot = 'Ch {}'.format(i)
+ axarr[i].set_ylabel(channel_num_plot)
- if name is None:
- name = self.file_name if self.file_name is not None else self._NAME_STEM
- else:
- name = os.path.splitext(name)[0]
+ if title is None:
+ title = self.file_name if self.file_name is not None else self._NAME_STEM
- plt.suptitle(name)
- name = name if self._check_if_valid_img_type(name) else name + '.png'
+ plt.suptitle(title)
- if save:
- output_name_path = output_path if output_path else ''
- output_name_path += name
- plt.savefig(output_name_path)
+ if file_path_name:
+ file_path_name = file_path_name if self._check_if_valid_img_type(file_path_name) \
+ else file_path_name + '.png'
+ plt.savefig(file_path_name)
+
+ plt.show()
def plot_spectrogram(self, file_name=None, ch=None):
# TODO: use self.stft_data if not None
@@ -812,7 +809,7 @@ class AudioSignal(object):
def _check_if_valid_img_type(name):
import matplotlib.pyplot as plt
fig = plt.figure()
- result = any([name[len(k):] == k for k in fig.canvas.get_supported_filetypes().keys()])
+ result = any([name[-len(k):] == k for k in fig.canvas.get_supported_filetypes().keys()])
plt.close()
return result
@@ -1170,13 +1167,14 @@ class AudioSignal(object):
Resample an audio signal using resample
Args:
new_sample_rate: (int) The new sample rate to apply to the audio signal
- overwrite: (bool) should overwrite audio_data in self
-
- Returns:
"""
- resampled_signal = librosa.resample(self.audio_data, self.sample_rate, new_sample_rate)
- self.audio_data = resampled_signal
+
+ resampled_signal = []
+ for channel in self.get_channels():
+ resampled_channel = librosa.resample(channel, self.sample_rate, new_sample_rate)
+ resampled_signal.append(resampled_channel)
+ self.audio_data = np.array(resampled_signal)
self._sample_rate = new_sample_rate
##################################################

diff:
@@ -7,6 +7,7 @@ import nussl
import numpy as np
import scipy.io.wavfile as wav
import os
+import librosa
import warnings
@@ -181,12 +182,14 @@ class AudioSignalUnitTests(unittest.TestCase):
sine_wave = np.sin(np.linspace(0, freq * 2 * np.pi, length))
def test_resample(self):
+ # Check that sample rate property changes
a = nussl.AudioSignal(self.input_path1)
b = nussl.AudioSignal(self.input_path1)
b.resample(a.sample_rate/2)
assert (b.sample_rate == a.sample_rate/2)
def test_resample_on_load_from_file(self):
+ # Test resample right when loading from file vs resampling after loading
a = nussl.AudioSignal(self.input_path1)
a.resample(48000)
b = nussl.AudioSignal()
@@ -194,13 +197,27 @@ class AudioSignalUnitTests(unittest.TestCase):
assert (a.sample_rate== b.sample_rate)
assert (np.allclose(a.audio_data, b.audio_data))
- def test_resample_on_load_from_array(self):
+ def test_resample_vs_librosa_load(self):
+ # Check against librosa load function
+ a = nussl.AudioSignal(self.input_path1)
+ a.resample(48000)
+ b_audio_data, b_sample_rate = librosa.load(self.input_path1, sr=48000)
+ assert (a.sample_rate== b_sample_rate)
+ assert (np.allclose(a.audio_data, b_audio_data))
+
+ def test_default_sr_on_load_from_array(self):
+ # Check that the default sample rate is set when no sample rate is provided load_audio_from_array
sr, data = wav.read(self.input_path1)
a = nussl.AudioSignal()
- a.load_audio_from_array(data, sample_rate=sr)
- a.resample(48000)
+ a.load_audio_from_array(data)
+ assert(a.sample_rate == nussl.constants.DEFAULT_SAMPLE_RATE)
+
+ def test_sr_on_load_from_array(self):
+ # Check that the passed in sample rate is being set in load_audio_from_array
+ a = nussl.AudioSignal(self.input_path1)
+ sr, data = wav.read(self.input_path1)
b = nussl.AudioSignal()
- b.load_audio_from_array(data, sample_rate=sr, new_sample_rate=48000)
+ b.load_audio_from_array(data, sample_rate=sr)
assert (a.sample_rate == b.sample_rate)
assert (np.allclose(a.audio_data, b.audio_data))
@@ -222,9 +239,9 @@ class AudioSignalUnitTests(unittest.TestCase):
def test_plot_time_domain_multi_channel(self):
# Plotting a signal with 5 channels
- num_test_channels = 6
+ num_test_channels = 5
freq_multiple = 5
- freqs = [i * freq_multiple for i in range(1, num_test_channels)]
+ freqs = [i * freq_multiple for i in range(1, num_test_channels+1)]
test_signal = np.array([np.sin(np.linspace(0, i * 2 * np.pi, self.length)) for i in freqs[:num_test_channels]])
a = nussl.AudioSignal(audio_data_array = test_signal)
a.plot_time_domain()
@@ -237,7 +254,7 @@ class AudioSignalUnitTests(unittest.TestCase):
def test_plot_time_domain_save_to_path_and_rename(self):
# Plotting and saving the plot with a new name to a folder
a = nussl.AudioSignal(self.input_path3)
- a.plot_time_domain(save=True, name='Test Graph', output_path='../')
+ a.plot_time_domain(file_path_name='test_graph.png')
def test_stft_istft_simple1(self):
"""


message: Adding first draft of NMF MFCC with tests and inputs.

change_type: ADD

diff:
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import nussl
+import numpy as np
+
+def main():
+ # Load input file
+
+
+ input_file_name = os.path.join('..', 'input', 'dev1_female3_inst_mix.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up DUET
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources= 3, num_templates=60, distance_measure="euclidean", num_iterations=50)
+
+ # and run
+ nmf_mfcc.run()
+ nmf_mfcc.make_audio_signals()
+
+if __name__ == '__main__':
+ main()

diff:
+e47266ac3908483ff4e2bd656a63d03aff94fe7d
\ No newline at end of file

diff:
+3bf00a74d886d6ba178d7272dcb32d8d04159514
\ No newline at end of file

diff:
@@ -15,10 +15,12 @@ from .duet import Duet
from .projet import Projet
# Others
+from .nmf_mfcc import NMF_MFCC
from .ideal_mask import IdealMask
from .overlap_add import OverlapAdd
from .ica import ICA
from .high_low_pass_filter import HighLowPassFilter
__all__ = ['SeparationBase', 'MaskSeparationBase',
- 'Repet', 'RepetSim', 'FT2D', 'Duet', 'Projet', 'IdealMask', 'OverlapAdd', 'ICA', 'HighLowPassFilter']
+ 'Repet', 'RepetSim', 'FT2D', 'Duet', 'Projet', 'IdealMask', 'OverlapAdd',
+ 'ICA', 'HighLowPassFilter', 'NMF_MFCC']

diff:
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+from sklearn.cluster import KMeans
+import librosa
+import nussl.audio_signal
+import nussl.constants
+import nussl.spectral_utils
+import nussl.utils
+from nussl.transformers import transformer_nmf
+from nussl.separation import separation_base
+import mask_separation_base
+import masks
+
+
+class NMF_MFCC(mask_separation_base.MaskSeparationBase):
+ def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure, num_iterations):
+ super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal,
+ mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
+
+ self.num_sources = num_sources
+ self.num_templates = num_templates
+ self.distance_measure = distance_measure
+ self.num_iterations = num_iterations
+ self.input_audio_signal = input_audio_signal
+ self.audio_data = input_audio_signal.audio_data
+ self.clusterer = None
+ self.signal_stft = None
+ self.templates_matrix = None
+ self.activation_matrix = None
+ self.labeled_templates = None
+ self.sources = None
+ self.masks = []
+ self.clusterer = KMeans(n_clusters=self.num_sources)
+ self.input_audio_signal.stft_params = self.stft_params
+ self.signal_stft = self.input_audio_signal.stft()
+
+ def run(self):
+ templates_stft = self.signal_stft[:, 0:self.signal_stft.shape[1]]
+ templates_stft = np.squeeze(templates_stft)
+
+ # Set up NMF and run
+ nmf = transformer_nmf.TransformerNMF(np.abs(templates_stft), self.num_templates)
+ nmf.should_use_epsilon = False
+ nmf.max_num_iterations = self.num_iterations
+ nmf.distance_measure = self.distance_measure
+ self.activation_matrix, self.templates_matrix = nmf.transform()
+
+ # Cluster the templates matrix into Mel frequencies
+ cluster_templates = librosa.feature.mfcc(S=self.templates_matrix)[1:14]
+ self.clusterer.fit_transform(cluster_templates.T)
+
+ # Retrieve cluster labels for the templates
+ self.labeled_templates = self.clusterer.labels_
+ self.sources = []
+
+ # Extract sources from signal
+ for source_index in range(self.num_sources):
+ source_indices = np.where(self.labeled_templates == source_index)[0]
+ self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
+ signal_stft=self.signal_stft)
+ return self.masks
+
+ def extract_masks(self, clustered_templates, signal_stft):
+ """ Extracts the binary mask objects for each source.
+ Parameters:
+ clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
+ source being extracted
+ signal_stft (np.matrix): The stft of the input audio signal
+ """
+ new_templates_stft = np.abs(np.squeeze(signal_stft))
+
+ # Set up and run NMF using each clustered template
+ reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
+ num_components=clustered_templates.shape[1], should_update_template=False)
+ reconstruct_nmf.should_use_epsilon = False
+ reconstruct_nmf.max_num_iterations = self.num_iterations
+ reconstruct_nmf.distance_measure = self.distance_measure
+ activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
+
+ # Reconstruct the signal
+ reconstructed_signal = new_templates_matrix.dot(activation_matrix)
+
+ # Mask the input signal
+ music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
+ mask = np.divide(reconstructed_signal, music_stft_max)
+ mask = np.nan_to_num(mask)
+
+ # Create the binary mask
+ mask = np.round(mask)
+ mask_object = masks.BinaryMask(np.array(mask))
+ self.masks.append(mask_object)
+
+ def make_audio_signals(self):
+ """ Applies each mask in self.masks and returns a list of audio_signal objects for each source.
+
+ Returns:
+ self.sources (np.array): An array of audio_signal objects containing each separated source
+ """
+ for i in range(self.num_sources):
+ source_audio_signal = self.audio_signal.make_copy_with_stft_data(np.squeeze(self.signal_stft), verbose=False)
+ source_audio_signal = source_audio_signal.apply_mask(self.masks[i])
+ source_audio_signal.stft_params = self.stft_params
+ source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
+ self.sources.append(source_audio_signal)
+ return self.sources
+
+ # # template - extracted template, residual - everything that's leftover.
+ # template = np.multiply(np.squeeze(signal_stft), mask)
+ # residual = np.multiply(np.squeeze(signal_stft), 1 - mask)
+ #
+ # return template, residual
\ No newline at end of file

diff:
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+import numpy as np
+import nussl
+import os
+
+
+
+class NMFMFCCUnitTests(unittest.TestCase):
+
+ def test_2_sin_waves(self):
+ sr = nussl.constants.DEFAULT_SAMPLE_RATE
+ dur = 3 # seconds
+ length = dur * sr
+ sine_wave_1 = np.sin(np.linspace(0, 440 * 2 * np.pi, length))
+ sine_wave_2 = np.sin(np.linspace(0, 440 * 5 * 2 * np.pi, length))
+ signal = sine_wave_1 + sine_wave_2
+ test_audio = nussl.AudioSignal()
+ test_audio.load_audio_from_array(signal)
+
+ # Set up NMMF MFCC
+ nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, distance_measure="euclidean",
+ num_iterations=50)
+
+ # and run
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
+
+ def test_piano_and_synth(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up NMMF MFCC
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=150)
+ # and run
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)


message: Removed second NMF to optimize running time and added benchmarks for algorithm with 2 NMFs.

diff:
@@ -37,6 +37,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.signal_stft = self.input_audio_signal.stft()
def run(self):
+ #TODO expose random seeding for NMF and KMeans
templates_stft = self.signal_stft[:, 0:self.signal_stft.shape[1]]
templates_stft = np.squeeze(templates_stft)
@@ -45,53 +46,64 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
nmf.should_use_epsilon = False
nmf.max_num_iterations = self.num_iterations
nmf.distance_measure = self.distance_measure
- self.activation_matrix, self.templates_matrix = nmf.transform()
+ self.activation_matrix, self.templates_matrix = nmf.transform() # FREEZE!
- # Cluster the templates matrix into Mel frequencies
+ # Cluster the templates matrix into Mel frequencies and retrieve labels
cluster_templates = librosa.feature.mfcc(S=self.templates_matrix)[1:14]
self.clusterer.fit_transform(cluster_templates.T)
-
- # Retrieve cluster labels for the templates
- self.labeled_templates = self.clusterer.labels_
- self.sources = []
+ self.labeled_templates = self.clusterer.labels_ # FREEZE!
# Extract sources from signal
+ self.sources = []
for source_index in range(self.num_sources):
source_indices = np.where(self.labeled_templates == source_index)[0]
- self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
- signal_stft=self.signal_stft)
- return self.masks
-
- def extract_masks(self, clustered_templates, signal_stft):
- """ Extracts the binary mask objects for each source.
- Parameters:
- clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
- source being extracted
- signal_stft (np.matrix): The stft of the input audio signal
- """
- new_templates_stft = np.abs(np.squeeze(signal_stft))
-
- # Set up and run NMF using each clustered template
- reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
- num_components=clustered_templates.shape[1], should_update_template=False)
- reconstruct_nmf.should_use_epsilon = False
- reconstruct_nmf.max_num_iterations = self.num_iterations
- reconstruct_nmf.distance_measure = self.distance_measure
- activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
-
- # Reconstruct the signal
- reconstructed_signal = new_templates_matrix.dot(activation_matrix)
-
- # Mask the input signal
- music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
- mask = np.divide(reconstructed_signal, music_stft_max)
- mask = np.nan_to_num(mask)
-
- # Create the binary mask
+ templates_mask = np.copy(self.templates_matrix)
+ activation_mask = np.copy(self.activation_matrix)
+ for i in range(templates_mask.shape[1]):
+ templates_mask[:, i] = 0 if i in source_indices else self.templates_matrix[:, i]
+ activation_mask[i, :] = 0 if i in source_indices else self.activation_matrix[i, :]
+ mask_matrix = templates_mask.dot(activation_mask)
+ music_stft_max = np.maximum(mask_matrix, np.abs(np.squeeze(self.signal_stft)))
+ mask_matrix = np.divide(mask_matrix, music_stft_max)
+ mask = np.nan_to_num(mask_matrix)
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
self.masks.append(mask_object)
+ # self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
+ # signal_stft=self.signal_stft)
+ return self.masks # FREEZE!
+
+ # def extract_masks(self, clustered_templates, signal_stft):
+ # """ Extracts the binary mask objects for each source.
+ # Parameters:
+ # clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
+ # source being extracted
+ # signal_stft (np.matrix): The stft of the input audio signal
+ # """
+ # new_templates_stft = np.abs(np.squeeze(signal_stft)) # TODO: multichannel
+ #
+ # # Set up and run NMF using each clustered template
+ # reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
+ # num_components=clustered_templates.shape[1], should_update_template=False)
+ # reconstruct_nmf.should_use_epsilon = False
+ # reconstruct_nmf.max_num_iterations = self.num_iterations
+ # reconstruct_nmf.distance_measure = self.distance_measure
+ # activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
+ #
+ # # Reconstruct the signal
+ # reconstructed_signal = new_templates_matrix.dot(activation_matrix)
+ #
+ # # Mask the input signal
+ # music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
+ # mask = np.divide(reconstructed_signal, music_stft_max)
+ # mask = np.nan_to_num(mask)
+ #
+ # # Create the binary mask
+ # mask = np.round(mask)
+ # mask_object = masks.BinaryMask(np.array(mask))
+ # self.masks.append(mask_object)
+
def make_audio_signals(self):
""" Applies each mask in self.masks and returns a list of audio_signal objects for each source.
@@ -104,7 +116,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
source_audio_signal.stft_params = self.stft_params
source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
self.sources.append(source_audio_signal)
- return self.sources
+ return self.sources # FREEZE!
# # template - extracted template, residual - everything that's leftover.
# template = np.multiply(np.squeeze(signal_stft), mask)

diff:
+5ee5d25fb0e0d4e73bd13e7a76e10ea77cc60ce9
\ No newline at end of file

diff:
+e80ca319a52501f5b340d3d34821120f19a790d5
\ No newline at end of file

diff:
@@ -202,7 +202,7 @@ def freeze_duet_values():
signal = nussl.AudioSignal(path)
duet = nussl.Duet(signal, 3)
- output_folder = os.path.abspath('duet_reference/test')
+ output_folder = os.path.abspath('duet_reference/duet_benchmarks')
duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)

diff:
@@ -10,6 +10,9 @@ import os
class NMFMFCCUnitTests(unittest.TestCase):
+ # Update this if the benchmark file changes and rerun freeze_duet_values() (below)
+ path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
+
def test_2_sin_waves(self):
sr = nussl.constants.DEFAULT_SAMPLE_RATE
dur = 3 # seconds
@@ -22,7 +25,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, distance_measure="euclidean",
- num_iterations=50)
+ num_iterations=5)
# and run
nmf_mfcc.run()
@@ -46,3 +49,22 @@ class NMFMFCCUnitTests(unittest.TestCase):
for i, source in enumerate(sources):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
+
+def freeze_nmf_mfcc_values():
+ path = NMFMFCCUnitTests.path_to_benchmark_file
+
+ signal = nussl.AudioSignal(path)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=150)
+
+ output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
+
+ nmf_mfcc.run()
+ np.save(os.path.join(output_folder, "benchmark_activation_matrix"), nmf_mfcc.activation_matrix)
+ np.save(os.path.join(output_folder, "benchmark_templates_matrix"), nmf_mfcc.templates_matrix)
+ np.save(os.path.join(output_folder, "benchmark_labeled_templates"), nmf_mfcc.labeled_templates)
+ np.save(os.path.join(output_folder, "benchmark_masks"), nmf_mfcc.masks)
+ nmf_mfcc.make_audio_signals()
+ np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
+
+freeze_nmf_mfcc_values()
\ No newline at end of file


message: Added freezing functions as their files for DUET and NMF MFCC.

diff:
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import nussl
+import os
+
+#Freeze key NMF MFCC values to the benchmark folder
+def freeze_nmf_mfcc_values():
+ path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
+ signal = nussl.AudioSignal(path_to_benchmark_file)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=150)
+ output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
+
+ nmf_mfcc.run()
+ np.save(os.path.join(output_folder, "benchmark_activation_matrix"), nmf_mfcc.activation_matrix)
+ np.save(os.path.join(output_folder, "benchmark_templates_matrix"), nmf_mfcc.templates_matrix)
+ np.save(os.path.join(output_folder, "benchmark_labeled_templates"), nmf_mfcc.labeled_templates)
+ np.save(os.path.join(output_folder, "benchmark_masks"), nmf_mfcc.masks)
+
+ nmf_mfcc.make_audio_signals()
+ np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
+
+if __name__ == '__main__':
+ freeze_nmf_mfcc_values()

diff:
@@ -70,40 +70,8 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
mask_object = masks.BinaryMask(np.array(mask))
self.masks.append(mask_object)
- # self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
- # signal_stft=self.signal_stft)
return self.masks # FREEZE!
- # def extract_masks(self, clustered_templates, signal_stft):
- # """ Extracts the binary mask objects for each source.
- # Parameters:
- # clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
- # source being extracted
- # signal_stft (np.matrix): The stft of the input audio signal
- # """
- # new_templates_stft = np.abs(np.squeeze(signal_stft)) # TODO: multichannel
- #
- # # Set up and run NMF using each clustered template
- # reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
- # num_components=clustered_templates.shape[1], should_update_template=False)
- # reconstruct_nmf.should_use_epsilon = False
- # reconstruct_nmf.max_num_iterations = self.num_iterations
- # reconstruct_nmf.distance_measure = self.distance_measure
- # activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
- #
- # # Reconstruct the signal
- # reconstructed_signal = new_templates_matrix.dot(activation_matrix)
- #
- # # Mask the input signal
- # music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
- # mask = np.divide(reconstructed_signal, music_stft_max)
- # mask = np.nan_to_num(mask)
- #
- # # Create the binary mask
- # mask = np.round(mask)
- # mask_object = masks.BinaryMask(np.array(mask))
- # self.masks.append(mask_object)
-
def make_audio_signals(self):
""" Applies each mask in self.masks and returns a list of audio_signal objects for each source.
@@ -117,9 +85,3 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
self.sources.append(source_audio_signal)
return self.sources # FREEZE!
-
- # # template - extracted template, residual - everything that's leftover.
- # template = np.multiply(np.squeeze(signal_stft), mask)
- # residual = np.multiply(np.squeeze(signal_stft), 1 - mask)
- #
- # return template, residual
\ No newline at end of file

diff:
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import division
+
+import unittest
+import nussl
+import numpy as np
+import scipy.io.wavfile as wav
+import scipy.io
+import os
+import warnings
+
+def freeze_duet_values():
+ signal = nussl.AudioSignal(os.path.join('..', 'Input', 'dev1_female3_inst_mix.wav'))
+ duet = nussl.Duet(signal, 3)
+ output_folder = os.path.abspath('duet_reference/duet_benchmarks')
+
+ duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
+ np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)
+ np.save(os.path.join(output_folder, "benchmark_stft_ch1"), duet.stft_ch1)
+ np.save(os.path.join(output_folder, "benchmark_wmat"), duet.frequency_matrix)
+
+ duet.symmetric_atn, duet.delay = duet._compute_atn_delay(duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix)
+ np.save(os.path.join(output_folder, "benchmark_sym_atn"), duet.symmetric_atn)
+ np.save(os.path.join(output_folder, "benchmark_delay"), duet.delay)
+
+ duet.normalized_attenuation_delay_histogram, duet.attenuation_bins, duet.delay_bins = duet._make_histogram()
+ np.save(os.path.join(output_folder, "benchmark_hist"), duet.normalized_attenuation_delay_histogram)
+ np.save(os.path.join(output_folder, "benchmark_atn_bins"), duet.attenuation_bins)
+ np.save(os.path.join(output_folder, "benchmark_delay_bins"), duet.delay_bins)
+
+ duet.peak_indices = nussl.utils.find_peak_indices(duet.normalized_attenuation_delay_histogram, duet.num_sources,
+ threshold=duet.peak_threshold,
+ min_dist=[duet.attenuation_min_distance,
+ duet.delay_min_distance])
+ np.save(os.path.join(output_folder,"benchmark_peak_indices"), duet.peak_indices)
+
+ duet.delay_peak, duet.atn_delay_est, duet.atn_peak = duet._convert_peaks()
+ np.save(os.path.join(output_folder, "benchmark_delay_peak"), duet.delay_peak)
+ np.save(os.path.join(output_folder, "benchmark_atn_delay_est"), duet.atn_delay_est)
+ np.save(os.path.join(output_folder, "benchmark_atn_peak"), duet.atn_peak)
+
+ duet.masks = duet._compute_masks()
+ np.save(os.path.join(output_folder, "benchmark_masks"), duet.masks)
+
+ final_signals = duet.make_audio_signals()
+ np.save(os.path.join(output_folder, "benchmark_final_signals"), final_signals)
+
+if __name__ == '__main__':
+ freeze_duet_values()
\ No newline at end of file

diff:
@@ -197,46 +197,5 @@ class DuetUnitTests(unittest.TestCase):
# TODO: fix this function before writing test for it
-def freeze_duet_values():
- path = DuetUnitTests.path_to_benchmark_file
-
- signal = nussl.AudioSignal(path)
- duet = nussl.Duet(signal, 3)
- output_folder = os.path.abspath('duet_reference/duet_benchmarks')
-
- duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
- np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)
- np.save(os.path.join(output_folder, "benchmark_stft_ch1"), duet.stft_ch1)
- np.save(os.path.join(output_folder, "benchmark_wmat"), duet.frequency_matrix)
-
- duet.symmetric_atn, duet.delay = duet._compute_atn_delay(duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix)
- np.save(os.path.join(output_folder, "benchmark_sym_atn"), duet.symmetric_atn)
- np.save(os.path.join(output_folder, "benchmark_delay"), duet.delay)
-
- duet.normalized_attenuation_delay_histogram, duet.attenuation_bins, duet.delay_bins = duet._make_histogram()
- np.save(os.path.join(output_folder, "benchmark_hist"), duet.normalized_attenuation_delay_histogram)
- np.save(os.path.join(output_folder, "benchmark_atn_bins"), duet.attenuation_bins)
- np.save(os.path.join(output_folder, "benchmark_delay_bins"), duet.delay_bins)
-
- duet.peak_indices = nussl.utils.find_peak_indices(duet.normalized_attenuation_delay_histogram, duet.num_sources,
- threshold=duet.peak_threshold,
- min_dist=[duet.attenuation_min_distance,
- duet.delay_min_distance])
- np.save(os.path.join(output_folder,"benchmark_peak_indices"), duet.peak_indices)
-
- duet.delay_peak, duet.atn_delay_est, duet.atn_peak = duet._convert_peaks()
- np.save(os.path.join(output_folder, "benchmark_delay_peak"), duet.delay_peak)
- np.save(os.path.join(output_folder, "benchmark_atn_delay_est"), duet.atn_delay_est)
- np.save(os.path.join(output_folder, "benchmark_atn_peak"), duet.atn_peak)
-
- duet.masks = duet._compute_masks()
- np.save(os.path.join(output_folder, "benchmark_masks"), duet.masks)
-
- final_signals = duet.make_audio_signals()
- np.save(os.path.join(output_folder, "benchmark_final_signals"), final_signals)
-
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
-
-# freeze_duet_values()
-

diff:
@@ -6,8 +6,6 @@ import numpy as np
import nussl
import os
-
-
class NMFMFCCUnitTests(unittest.TestCase):
# Update this if the benchmark file changes and rerun freeze_duet_values() (below)
@@ -35,6 +33,17 @@ class NMFMFCCUnitTests(unittest.TestCase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
+ def load_benchmarks(self):
+ benchmark_dict = {}
+ directory = 'nmf_mfcc_reference/nmf_mfcc_benchmarks'
+ for filename in os.listdir(directory):
+ key = os.path.splitext(filename)[0]
+ file_path = os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', filename)
+ value = np.load(file_path)
+ benchmark_dict[key] = value
+ return benchmark_dict
+
+
def test_piano_and_synth(self):
# Load input file
input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
@@ -50,21 +59,28 @@ class NMFMFCCUnitTests(unittest.TestCase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
-def freeze_nmf_mfcc_values():
- path = NMFMFCCUnitTests.path_to_benchmark_file
+ def benchmark_nmf_mfcc(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
- signal = nussl.AudioSignal(path)
+ # Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
num_iterations=150)
+ # and run
+ nmf_mfcc.run()
- output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
+ benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_activation_matrix.npy'))
+ benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_templates_matrix.npy'))
+ benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_labeled_templates.npy'))
+ benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_masks.npy'))
- nmf_mfcc.run()
- np.save(os.path.join(output_folder, "benchmark_activation_matrix"), nmf_mfcc.activation_matrix)
- np.save(os.path.join(output_folder, "benchmark_templates_matrix"), nmf_mfcc.templates_matrix)
- np.save(os.path.join(output_folder, "benchmark_labeled_templates"), nmf_mfcc.labeled_templates)
- np.save(os.path.join(output_folder, "benchmark_masks"), nmf_mfcc.masks)
nmf_mfcc.make_audio_signals()
- np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
+ benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_sources.npy'))
+
+ assert np.all(benchmark_activation_matrix == nmf_mfcc.activation_matrix)
+ assert np.all(benchmark_templates_matrix == nmf_mfcc.templates_matrix)
+ assert np.all(benchmark_labeled_templates == nmf_mfcc.labeled_templates)
+ assert np.all(benchmark_masks == nmf_mfcc.masks)
+ assert np.all(benchmark_sources == nmf_mfcc.sources)
-freeze_nmf_mfcc_values()
\ No newline at end of file


message: Added multi-channel functionality for nmf_mfcc, exposed random seeding, and created tests.

change_type: RENAME

diff:
@@ -15,7 +15,8 @@ import masks
class NMF_MFCC(mask_separation_base.MaskSeparationBase):
- def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure, num_iterations):
+ def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure,
+ num_iterations, random_seed, mono):
super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal,
mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
@@ -23,6 +24,9 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.num_templates = num_templates
self.distance_measure = distance_measure
self.num_iterations = num_iterations
+ self.random_seed = None
+ self.mono=False
+
self.input_audio_signal = input_audio_signal
self.audio_data = input_audio_signal.audio_data
self.clusterer = None
@@ -32,45 +36,58 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.labeled_templates = None
self.sources = None
self.masks = []
+
+ if self.mono:
+ self.input_audio_signal.audio_data.to_mono(overwrite=True)
+
self.clusterer = KMeans(n_clusters=self.num_sources)
self.input_audio_signal.stft_params = self.stft_params
self.signal_stft = self.input_audio_signal.stft()
def run(self):
- #TODO expose random seeding for NMF and KMeans
- templates_stft = self.signal_stft[:, 0:self.signal_stft.shape[1]]
- templates_stft = np.squeeze(templates_stft)
+ for i in range(self.audio_signal.num_channels):
+ channel_stft = self.signal_stft[:, :, i]
# Set up NMF and run
- nmf = transformer_nmf.TransformerNMF(np.abs(templates_stft), self.num_templates)
+ nmf = transformer_nmf.TransformerNMF(input_matrix=np.abs(channel_stft), num_components=self.num_templates,
+ seed=self.random_seed)
nmf.should_use_epsilon = False
nmf.max_num_iterations = self.num_iterations
nmf.distance_measure = self.distance_measure
- self.activation_matrix, self.templates_matrix = nmf.transform() # FREEZE!
+ channel_activation_matrix, channel_templates_matrix = nmf.transform()
# Cluster the templates matrix into Mel frequencies and retrieve labels
- cluster_templates = librosa.feature.mfcc(S=self.templates_matrix)[1:14]
+ cluster_templates = librosa.feature.mfcc(S=channel_templates_matrix)[1:14]
self.clusterer.fit_transform(cluster_templates.T)
- self.labeled_templates = self.clusterer.labels_ # FREEZE!
+ self.labeled_templates = self.clusterer.labels_
# Extract sources from signal
+ channel_masks = self.extract_masks(channel_stft, channel_templates_matrix, channel_activation_matrix)
+ self.masks.append(channel_masks)
+
+ return self.masks
+
+ def extract_masks(self, signal_stft, templates_matrix, activation_matrix):
+ if signal_stft is None:
+ raise ValueError('Cannot extract masks with no signal_stft data')
+
self.sources = []
+ channel_mask_list = []
for source_index in range(self.num_sources):
source_indices = np.where(self.labeled_templates == source_index)[0]
- templates_mask = np.copy(self.templates_matrix)
- activation_mask = np.copy(self.activation_matrix)
+ templates_mask = np.copy(templates_matrix)
+ activation_mask = np.copy(activation_matrix)
for i in range(templates_mask.shape[1]):
- templates_mask[:, i] = 0 if i in source_indices else self.templates_matrix[:, i]
- activation_mask[i, :] = 0 if i in source_indices else self.activation_matrix[i, :]
+ templates_mask[:, i] = 0 if i in source_indices else templates_matrix[:, i]
+ activation_mask[i, :] = 0 if i in source_indices else activation_matrix[i, :]
mask_matrix = templates_mask.dot(activation_mask)
- music_stft_max = np.maximum(mask_matrix, np.abs(np.squeeze(self.signal_stft)))
+ music_stft_max = np.maximum(mask_matrix, np.abs(signal_stft))
mask_matrix = np.divide(mask_matrix, music_stft_max)
mask = np.nan_to_num(mask_matrix)
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
- self.masks.append(mask_object)
-
- return self.masks # FREEZE!
+ channel_mask_list.append(mask_object)
+ return channel_mask_list
def make_audio_signals(self):
""" Applies each mask in self.masks and returns a list of audio_signal objects for each source.
@@ -78,10 +95,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Returns:
self.sources (np.array): An array of audio_signal objects containing each separated source
"""
- for i in range(self.num_sources):
- source_audio_signal = self.audio_signal.make_copy_with_stft_data(np.squeeze(self.signal_stft), verbose=False)
- source_audio_signal = source_audio_signal.apply_mask(self.masks[i])
+ for i in range(self.audio_signal.num_channels):
+ channel_mask = self.masks[i]
+ for j in range(self.num_sources):
+ source_audio_signal = self.audio_signal.make_copy_with_stft_data((self.signal_stft[:,:,i]), verbose=False)
+ source_audio_signal = source_audio_signal.apply_mask(channel_mask[j])
source_audio_signal.stft_params = self.stft_params
source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
self.sources.append(source_audio_signal)
- return self.sources # FREEZE!
+ return self.sources

diff:
@@ -6,11 +6,11 @@ import nussl
import os
#Freeze key NMF MFCC values to the benchmark folder
-def freeze_nmf_mfcc_values():
+def main():
path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_benchmark_file)
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=150, random_seed=0)
output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
nmf_mfcc.run()
@@ -23,4 +23,4 @@ def freeze_nmf_mfcc_values():
np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
if __name__ == '__main__':
- freeze_nmf_mfcc_values()
+ main()

diff:
+650d69d03451df290fda2fff1ba0b59e23b23a89
\ No newline at end of file

diff:
@@ -33,6 +33,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
+
def load_benchmarks(self):
benchmark_dict = {}
directory = 'nmf_mfcc_reference/nmf_mfcc_benchmarks'
@@ -51,7 +52,22 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=10, random_seed=0, mono=False)
+ # and run
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
+
+ def test_piano_and_synth_stereo(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_stereo.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up NMMF MFCC
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=20, random_seed=0, mono=False)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
@@ -66,17 +82,17 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=150, random_seed=0)
# and run
nmf_mfcc.run()
- benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_activation_matrix.npy'))
- benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_templates_matrix.npy'))
- benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_labeled_templates.npy'))
- benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_masks.npy'))
+ benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_activation_matrix.npy'))
+ benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_templates_matrix.npy'))
+ benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_labeled_templates.npy'))
+ benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_masks.npy'))
nmf_mfcc.make_audio_signals()
- benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_sources.npy'))
+ benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_sources.npy'))
assert np.all(benchmark_activation_matrix == nmf_mfcc.activation_matrix)
assert np.all(benchmark_templates_matrix == nmf_mfcc.templates_matrix)
@@ -84,3 +100,4 @@ class NMFMFCCUnitTests(unittest.TestCase):
assert np.all(benchmark_masks == nmf_mfcc.masks)
assert np.all(benchmark_sources == nmf_mfcc.sources)
+


message: Added tests for NMF MFCC and first draft of documentation.

diff:
@@ -117,9 +117,7 @@ class Duet(mask_separation_base.MaskSeparationBase):
""" Extracts N sources from a given stereo audio mixture (N sources captured via 2 sensors)
Returns:
- source_estimates (np.array): an N-row matrix containing time-domain estimates of sources
- atn_delay_est (np.array): N by 2 Numpy matrix containing estimated attenuation and delay values
- corresponding to N sources
+ computed_masks (np.array): A list of binary mask objects that can be used to extract the sources
Example:
.. code-block:: python

diff:
@@ -15,8 +15,41 @@ import masks
class NMF_MFCC(mask_separation_base.MaskSeparationBase):
- def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure,
- num_iterations, random_seed, mono):
+ """
+ NMF MFCC (Non Negative Matrix Factorization using Mel Frequency Cepstral Coefficients) is a source separation
+ algorithm that runs Transformer NMF on the magnitude spectrogram of an input audio signal.
+ It uses K means clustering to cluster the templates and activations returned by the NMF. The dot product of the
+ clustered templates and activations result in a magnitude spectrogram only containing the separated source.
+ This is used to create a Binary Mask object, which can then be applied to return a list of Audio Signal objects
+ corresponding to each separated source.
+
+ References:
+
+ Parameters:
+ input_audio_signal (np.array): a 2-row Numpy matrix containing samples of the two-channel mixture.
+ num_sources (int): Number of sources to find.
+ num_templates (int): Number of template vectors to used in NMF.
+ distance_measure (str): The type of distance measure to use in NMF - euclidean or divergence.
+ num_iterations (int): The number of iterations to go through in NMF.
+ random_seed (int): The seed to use in the numpy random generator
+ convert_to_mono (bool): Given a stereo signal, convert to mono
+
+ Attributes:
+ input_audio_signal (object): An Audio Signal object of the input audio signal
+ clusterer (object): A KMeans object for clustering the templates and activations
+ signal_stft (np.matrix): The stft data for the current
+ templates_matrix (np.matrix): A Numpy matrix containing the templates matrix from running NMF on the
+ current channel from the input signal stft data.
+ activation_matrix (np.matrix): A Numpy matrix containing the activation matrix from running NMF on the
+ current channel from the input signal stft data.
+ labeled_templates (np.array): A Numpy array containing the labeled templates columns
+ from the templates matrix for a particular source
+ sources (np.array): A Numpy array containing the list of Audio Signal objects for each source
+ masks (np.array): A Numpy array containing the lists of Binary Mask objects for each channel
+
+ """
+ def __init__(self, input_audio_signal, num_sources, num_templates=50, distance_measure='euclidean',
+ num_iterations=50, random_seed=0, convert_to_mono=False):
super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal,
mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
@@ -24,11 +57,10 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.num_templates = num_templates
self.distance_measure = distance_measure
self.num_iterations = num_iterations
- self.random_seed = None
- self.mono=False
+ self.random_seed = random_seed
+ self.convert_to_mono = convert_to_mono
self.input_audio_signal = input_audio_signal
- self.audio_data = input_audio_signal.audio_data
self.clusterer = None
self.signal_stft = None
self.templates_matrix = None
@@ -37,15 +69,22 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.sources = None
self.masks = []
- if self.mono:
- self.input_audio_signal.audio_data.to_mono(overwrite=True)
+ # Convert the stereo signal to mono if indicated
+ if self.convert_to_mono:
+ self.input_audio_signal.to_mono(overwrite=True, remove_channels=False)
+ # Initialize the K Means clusterer
self.clusterer = KMeans(n_clusters=self.num_sources)
self.input_audio_signal.stft_params = self.stft_params
self.signal_stft = self.input_audio_signal.stft()
def run(self):
- for i in range(self.audio_signal.num_channels):
+ """ Extracts N sources from a given mixture
+
+ Returns:
+ self.masks (np.array): A list of binary mask objects that can be used to extract the sources
+ """
+ for i in range(self.input_audio_signal.num_channels):
channel_stft = self.signal_stft[:, :, i]
# Set up NMF and run
@@ -68,6 +107,18 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
return self.masks
def extract_masks(self, signal_stft, templates_matrix, activation_matrix):
+ """ Creates binary masks from clustered templates and activation matrices
+ Parameters:
+ signal_stft (np.matrix): A 2D Numpy matrix containing the stft of the current channel
+ templates_matrix (np.matrix): A 2D Numpy matrix containing the templates matrix after running NMF on
+ the current channel
+ activation_matrix (np.matrix): A 2D Numpy matrix containing the activation matrix after running NMF on
+ the current channel
+
+ Returns:
+ channel_mask_list (np.matrix): A list of Binary Mask objects corresponding to each source
+ """
+
if signal_stft is None:
raise ValueError('Cannot extract masks with no signal_stft data')
@@ -77,9 +128,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
source_indices = np.where(self.labeled_templates == source_index)[0]
templates_mask = np.copy(templates_matrix)
activation_mask = np.copy(activation_matrix)
+
+ # Zero out everything but the source determined from the clusterer
for i in range(templates_mask.shape[1]):
templates_mask[:, i] = 0 if i in source_indices else templates_matrix[:, i]
activation_mask[i, :] = 0 if i in source_indices else activation_matrix[i, :]
+
mask_matrix = templates_mask.dot(activation_mask)
music_stft_max = np.maximum(mask_matrix, np.abs(signal_stft))
mask_matrix = np.divide(mask_matrix, music_stft_max)
@@ -95,12 +149,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Returns:
self.sources (np.array): An array of audio_signal objects containing each separated source
"""
- for i in range(self.audio_signal.num_channels):
+ for i in range(self.input_audio_signal.num_channels):
channel_mask = self.masks[i]
for j in range(self.num_sources):
- source_audio_signal = self.audio_signal.make_copy_with_stft_data((self.signal_stft[:,:,i]), verbose=False)
- source_audio_signal = source_audio_signal.apply_mask(channel_mask[j])
- source_audio_signal.stft_params = self.stft_params
- source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
- self.sources.append(source_audio_signal)
+ source = self.audio_signal.make_copy_with_stft_data((self.signal_stft[:, :, i]), verbose=False)
+ source = source.apply_mask(channel_mask[j])
+ source.stft_params = self.stft_params
+ source.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
+ self.sources.append(source)
return self.sources

diff:
@@ -316,6 +316,24 @@ class AudioSignalUnitTests(unittest.TestCase):
assert (sig2.num_channels == 1)
assert (np.allclose([0.0] * len(sig2), sig2.audio_data))
+ def test_to_mono_channel_dimension(self):
+ """
+ Test functionality and correctness of AudioSignal.to_mono() function.
+ Returns:
+
+ """
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_stereo.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+ signal.stft_params = signal.stft_params
+ signal_stft = signal.stft()
+ assert (signal_stft.shape[2] == 2)
+
+ signal.to_mono(overwrite=True)
+ signal.stft_params = signal.stft_params
+ signal_stft = signal.stft()
+ assert (signal_stft.shape[2] == 1)
+
def test_stft(self):
"""
Test some basic functionality of the STFT interface for the AudioSignal object

diff:
@@ -23,7 +23,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, distance_measure="euclidean",
- num_iterations=5)
+ num_iterations=15)
# and run
nmf_mfcc.run()
@@ -52,7 +52,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=10, random_seed=0, mono=False)
+ num_iterations=10, random_seed=0)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
@@ -67,7 +67,22 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=20, random_seed=0, mono=False)
+ num_iterations=150, random_seed=0, convert_to_mono=False)
+ # and run
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
+
+ def test_piano_and_synth_stereo_to_mono(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_stereo.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up NMMF MFCC and convert input audio signal to mono
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=15, random_seed=0, convert_to_mono=True)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()


message: Added a demo for nmf_mfcc.

diff:
@@ -6,17 +6,22 @@ import numpy as np
def main():
# Load input file
-
-
- input_file_name = os.path.join('..', 'input', 'dev1_female3_inst_mix.wav')
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_input_file=input_file_name)
- # Set up DUET
- nmf_mfcc = nussl.NMF_MFCC(signal, num_sources= 3, num_templates=60, distance_measure="euclidean", num_iterations=50)
+ # make a directory to store output if needed
+ if not os.path.exists(os.path.join('..', 'Output/')):
+ os.mkdir(os.path.join('..', 'Output/'))
+ # Set up NMMF MFCC
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=10, random_seed=0)
# and run
nmf_mfcc.run()
- nmf_mfcc.make_audio_signals()
+ sources = nmf_mfcc.make_audio_signals()
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
if __name__ == '__main__':
main()

diff:
@@ -101,12 +101,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.labeled_templates = self.clusterer.labels_
# Extract sources from signal
- channel_masks = self.extract_masks(channel_stft, channel_templates_matrix, channel_activation_matrix)
+ channel_masks = self._extract_masks(channel_stft, channel_templates_matrix, channel_activation_matrix)
self.masks.append(channel_masks)
return self.masks
- def extract_masks(self, signal_stft, templates_matrix, activation_matrix):
+ def _extract_masks(self, signal_stft, templates_matrix, activation_matrix):
""" Creates binary masks from clustered templates and activation matrices
Parameters:
signal_stft (np.matrix): A 2D Numpy matrix containing the stft of the current channel

original_message: Added a demo for nmf_mfcc.


message: Re-froze benchmarks to include changes to sample rate.

diff:
-9679e0e6b03d248e033187695c5b4517bb4f0a79
\ No newline at end of file
+43845d5fe62a253a6f83d8c49734ac81994f3cb8
\ No newline at end of file

diff:
@@ -188,42 +188,9 @@ class DuetUnitTests(unittest.TestCase):
final_signals = duet.make_audio_signals()
- assert np.all(benchmark_final_signals == final_signals)
+ # Is the audio data the same?
+ assert all(np.array_equal(benchmark_final_signals[i].audio_data, final_signals[i].audio_data)
+ for i in range(len(final_signals)))
-def freeze_duet_values():
- path = DuetUnitTests.path_to_benchmark_file
-
- signal = nussl.AudioSignal(path)
- duet = nussl.Duet(signal, 3)
- output_folder = os.path.abspath('duet_reference/test')
-
- duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
- np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)
- np.save(os.path.join(output_folder, "benchmark_stft_ch1"), duet.stft_ch1)
- np.save(os.path.join(output_folder, "benchmark_wmat"), duet.frequency_matrix)
-
- duet.symmetric_atn, duet.delay = duet._compute_atn_delay(duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix)
- np.save(os.path.join(output_folder, "benchmark_sym_atn"), duet.symmetric_atn)
- np.save(os.path.join(output_folder, "benchmark_delay"), duet.delay)
-
- duet.normalized_attenuation_delay_histogram, duet.attenuation_bins, duet.delay_bins = duet._make_histogram()
- np.save(os.path.join(output_folder, "benchmark_hist"), duet.normalized_attenuation_delay_histogram)
- np.save(os.path.join(output_folder, "benchmark_atn_bins"), duet.attenuation_bins)
- np.save(os.path.join(output_folder, "benchmark_delay_bins"), duet.delay_bins)
-
- duet.peak_indices = nussl.utils.find_peak_indices(duet.normalized_attenuation_delay_histogram, duet.num_sources,
- threshold=duet.peak_threshold,
- min_dist=[duet.attenuation_min_distance,
- duet.delay_min_distance])
- np.save(os.path.join(output_folder, "benchmark_peak_indices"), duet.peak_indices)
-
- duet.delay_peak, duet.atn_delay_est, duet.atn_peak = duet._convert_peaks()
- np.save(os.path.join(output_folder, "benchmark_delay_peak"), duet.delay_peak)
- np.save(os.path.join(output_folder, "benchmark_atn_delay_est"), duet.atn_delay_est)
- np.save(os.path.join(output_folder, "benchmark_atn_peak"), duet.atn_peak)
-
- duet.result_masks = duet._compute_masks()
- np.save(os.path.join(output_folder, "benchmark_masks"), duet.result_masks)
-
- final_signals = duet.make_audio_signals()
- np.save(os.path.join(output_folder, "benchmark_final_signals"), final_signals)
+ # Check to see if AudioSignal's API changed; do we need to refreeze?
+ assert all(benchmark_final_signals[i] == final_signals[i] for i in range(len(final_signals)))


message: Exposed parameters for mfcc, random seeding, mask types, and added tests.

diff:
# -*- coding: utf-8 -*-
import numpy as np
-from sklearn.cluster import KMeans
+import sklearn.cluster
import librosa
import nussl.audio_signal
import nussl.constants
@@ -16,14 +16,15 @@ import masks
class NMF_MFCC(mask_separation_base.MaskSeparationBase):
"""
- NMF MFCC (Non Negative Matrix Factorization using Mel Frequency Cepstral Coefficients) is a source separation
+ Non Negative Matrix Factorization using K-Means Clustering on MFCC (NMF MFCC) is a source separation
algorithm that runs Transformer NMF on the magnitude spectrogram of an input audio signal.
It uses K means clustering to cluster the templates and activations returned by the NMF. The dot product of the
- clustered templates and activations result in a magnitude spectrogram only containing the separated source.
+ clustered templates and activations results in a magnitude spectrogram only containing the separated source.
This is used to create a Binary Mask object, which can then be applied to return a list of Audio Signal objects
corresponding to each separated source.
References:
+ Mel Frequency Cepstral Coefficients(MFCC): https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
Parameters:
input_audio_signal (np.array): a 2-row Numpy matrix containing samples of the two-channel mixture.
@@ -37,7 +38,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Attributes:
input_audio_signal (object): An Audio Signal object of the input audio signal
clusterer (object): A KMeans object for clustering the templates and activations
- signal_stft (np.matrix): The stft data for the current
+ signal_stft (np.matrix): The stft data for the input audio signal
templates_matrix (np.matrix): A Numpy matrix containing the templates matrix from running NMF on the
current channel from the input signal stft data.
activation_matrix (np.matrix): A Numpy matrix containing the activation matrix from running NMF on the
@@ -49,54 +50,85 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
"""
def __init__(self, input_audio_signal, num_sources, num_templates=50, distance_measure='euclidean',
- num_iterations=50, random_seed=0, convert_to_mono=False):
- super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal,
- mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
+ num_iterations=50, random_seed=None, kmeans_kwargs=None, convert_to_mono=False,
+ mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK, mfcc_range=(1, 14), n_mfcc=20):
+ super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal, mask_type=mask_type)
self.num_sources = num_sources
self.num_templates = num_templates
self.distance_measure = distance_measure
self.num_iterations = num_iterations
self.random_seed = random_seed
+ self.kmeans_kwargs = kmeans_kwargs
self.convert_to_mono = convert_to_mono
+ self.n_mfcc = n_mfcc
+ self.mask_type = mask_type
- self.input_audio_signal = input_audio_signal
- self.clusterer = None
self.signal_stft = None
+ self.input_audio_signal = input_audio_signal
self.templates_matrix = None
self.activation_matrix = None
self.labeled_templates = None
- self.sources = None
+ self.sources = []
self.masks = []
# Convert the stereo signal to mono if indicated
if self.convert_to_mono:
self.input_audio_signal.to_mono(overwrite=True, remove_channels=False)
+ # Set the MFCC range
+ if isinstance(mfcc_range, int) and mfcc_range < n_mfcc:
+ self.mfcc_start, self.mfcc_end = 1, mfcc_range
+ elif isinstance(mfcc_range, (tuple, list)) and len(mfcc_range) == 2:
+ self.mfcc_start, self.mfcc_end = mfcc_range[0], mfcc_range[1]
+ else:
+ raise ValueError('mfcc_range is not set correctly! Must be a tuple or list with min and max, or int (max)')
+
+ # If provided, add random_seed to kmeans_kwargs
+ if self.random_seed is not None:
+ if self.kmeans_kwargs is None:
+ self.kmeans_kwargs = {'random_state': self.random_seed}
+ if 'random_state' not in self.kmeans_kwargs or self.kmeans_kwargs['random_state'] is None:
+ self.kmeans_kwargs['random_state'] = self.random_seed
+
+ # Set number of clusters to number of sources
+ if self.kmeans_kwargs is None:
+ self.kmeans_kwargs = {'n_clusters': self.num_sources}
+ else:
+ self.kmeans_kwargs['n_clusters'] = self.num_sources
+
# Initialize the K Means clusterer
- self.clusterer = KMeans(n_clusters=self.num_sources)
- self.input_audio_signal.stft_params = self.stft_params
- self.signal_stft = self.input_audio_signal.stft()
+ self.clusterer = sklearn.cluster.KMeans(**self.kmeans_kwargs)
def run(self):
- """ Extracts N sources from a given mixture
+ """ This function calls TransformerNMF on the magnitude spectrogram of each channel in the input audio signal.
+ The templates and activation matrices returned are clustered using K-Means clustering. These clusters are used
+ to create mask objects for each source. Note: The masks are not returned in a particular order, but they are in
+ the same order for each channel.
Returns:
self.masks (np.array): A list of binary mask objects that can be used to extract the sources
"""
+ self.masks = []
+ self.input_audio_signal.stft_params = self.stft_params
+ self.signal_stft = self.input_audio_signal.stft()
+
for i in range(self.input_audio_signal.num_channels):
- channel_stft = self.signal_stft[:, :, i]
+ channel_stft = self.input_audio_signal.get_magnitude_spectrogram_channel(i)
# Set up NMF and run
- nmf = transformer_nmf.TransformerNMF(input_matrix=np.abs(channel_stft), num_components=self.num_templates,
+ nmf = transformer_nmf.TransformerNMF(input_matrix=channel_stft, num_components=self.num_templates,
seed=self.random_seed)
+
+ # Pass these to constructor (above)
nmf.should_use_epsilon = False
nmf.max_num_iterations = self.num_iterations
nmf.distance_measure = self.distance_measure
channel_activation_matrix, channel_templates_matrix = nmf.transform()
# Cluster the templates matrix into Mel frequencies and retrieve labels
- cluster_templates = librosa.feature.mfcc(S=channel_templates_matrix)[1:14]
+ cluster_templates = librosa.feature.mfcc(S=channel_templates_matrix,
+ n_mfcc=self.n_mfcc)[self.mfcc_start:self.mfcc_end]
self.clusterer.fit_transform(cluster_templates.T)
self.labeled_templates = self.clusterer.labels_
@@ -108,6 +140,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
def _extract_masks(self, signal_stft, templates_matrix, activation_matrix):
""" Creates binary masks from clustered templates and activation matrices
+
Parameters:
signal_stft (np.matrix): A 2D Numpy matrix containing the stft of the current channel
templates_matrix (np.matrix): A 2D Numpy matrix containing the templates matrix after running NMF on
@@ -138,8 +171,11 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
music_stft_max = np.maximum(mask_matrix, np.abs(signal_stft))
mask_matrix = np.divide(mask_matrix, music_stft_max)
mask = np.nan_to_num(mask_matrix)
+ if self.mask_type == self.BINARY_MASK:
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
+ else:
+ mask_object = masks.SoftMask(np.array(mask))
channel_mask_list.append(mask_object)
return channel_mask_list
@@ -149,10 +185,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Returns:
self.sources (np.array): An array of audio_signal objects containing each separated source
"""
+ self.sources = []
for i in range(self.input_audio_signal.num_channels):
channel_mask = self.masks[i]
for j in range(self.num_sources):
- source = self.audio_signal.make_copy_with_stft_data((self.signal_stft[:, :, i]), verbose=False)
+ channel_stft = self.input_audio_signal.get_stft_channel(i)
+ source = self.audio_signal.make_copy_with_stft_data(channel_stft, verbose=False)
source = source.apply_mask(channel_mask[j])
source.stft_params = self.stft_params
source.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)

diff:
@@ -11,6 +11,61 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Update this if the benchmark file changes and rerun freeze_duet_values() (below)
path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
+ def load_benchmarks(self):
+ benchmark_dict = {}
+ directory = 'nmf_mfcc_reference/nmf_mfcc_benchmarks'
+ for filename in os.listdir(directory):
+ key = os.path.splitext(filename)[0]
+ file_path = os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', filename)
+ value = np.load(file_path)
+ benchmark_dict[key] = value
+ return benchmark_dict
+
+ def test_mfcc_initialization(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up NMMF MFCC using an int
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=5, num_iterations=5)
+ assert nmf_mfcc.mfcc_start == 1
+ assert nmf_mfcc.mfcc_end == 5
+
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+
+ # Using a list
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=[3, 15], num_iterations=5)
+ assert nmf_mfcc.mfcc_start == 3
+ assert nmf_mfcc.mfcc_end == 15
+
+ # Using a tuple
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=(2, 14), num_iterations=5)
+ assert nmf_mfcc.mfcc_start == 2
+ assert nmf_mfcc.mfcc_end == 14
+
+ def test_random_seed_initialization(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Differing random_seed and random_state, check if set separately
+ kmean_kwargs = {'random_state': 0}
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, num_iterations=5, random_seed=1,
+ kmeans_kwargs=kmean_kwargs)
+ assert nmf_mfcc.clusterer.random_state == 0
+ assert nmf_mfcc.random_seed == 1
+
+ # No random_seed, but random_state initialized
+ kmean_kwargs = {'random_state': 0}
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, num_iterations=5, kmeans_kwargs=kmean_kwargs)
+ assert nmf_mfcc.clusterer.random_state == 0
+ assert nmf_mfcc.random_seed == None
+
+ # No random_state, but random_seed initialized
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, num_iterations=5, random_seed=0)
+ assert nmf_mfcc.clusterer.random_state == nmf_mfcc.random_seed == 0
+
def test_2_sin_waves(self):
sr = nussl.constants.DEFAULT_SAMPLE_RATE
dur = 3 # seconds
@@ -22,8 +77,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
test_audio.load_audio_from_array(signal)
# Set up NMMF MFCC
- nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, distance_measure="euclidean",
- num_iterations=15)
+ nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, num_iterations=15)
# and run
nmf_mfcc.run()
@@ -33,26 +87,13 @@ class NMFMFCCUnitTests(unittest.TestCase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
-
- def load_benchmarks(self):
- benchmark_dict = {}
- directory = 'nmf_mfcc_reference/nmf_mfcc_benchmarks'
- for filename in os.listdir(directory):
- key = os.path.splitext(filename)[0]
- file_path = os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', filename)
- value = np.load(file_path)
- benchmark_dict[key] = value
- return benchmark_dict
-
-
def test_piano_and_synth(self):
# Load input file
input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_input_file=input_file_name)
# Set up NMMF MFCC
- nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=10, random_seed=0)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, num_iterations=50, random_seed=0)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
@@ -66,8 +107,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
signal = nussl.AudioSignal(path_to_input_file=input_file_name)
# Set up NMMF MFCC
- nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150, random_seed=0, convert_to_mono=False)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, num_iterations=150, random_seed=0)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
@@ -101,16 +141,12 @@ class NMFMFCCUnitTests(unittest.TestCase):
# and run
nmf_mfcc.run()
- benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_activation_matrix.npy'))
- benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_templates_matrix.npy'))
benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_labeled_templates.npy'))
benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_masks.npy'))
nmf_mfcc.make_audio_signals()
benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_sources.npy'))
- assert np.all(benchmark_activation_matrix == nmf_mfcc.activation_matrix)
- assert np.all(benchmark_templates_matrix == nmf_mfcc.templates_matrix)
assert np.all(benchmark_labeled_templates == nmf_mfcc.labeled_templates)
assert np.all(benchmark_masks == nmf_mfcc.masks)
assert np.all(benchmark_sources == nmf_mfcc.sources)


message: Made edits to tests, random seed, documentation.

diff:
@@ -6,9 +6,13 @@ Base class for separation algorithms that make masks. Most algorithms in nussl a
"""
import warnings
+import json
import separation_base
import masks
+import nussl.constants
+import nussl.utils
+import nussl.audio_signal
class MaskSeparationBase(separation_base.SeparationBase):
@@ -49,6 +53,7 @@ class MaskSeparationBase(separation_base.SeparationBase):
self.mask_type = mask_type
self._mask_threshold = None
self.mask_threshold = mask_threshold
+ self.result_masks = []
@property
def mask_type(self):
@@ -166,7 +171,7 @@ class MaskSeparationBase(separation_base.SeparationBase):
@mask_threshold.setter
def mask_threshold(self, value):
- if not isinstance(value, float) or not (0.0 < value and value < 1.0):
+ if not isinstance(value, float) or not (0.0 < value < 1.0):
raise ValueError('Mask threshold must be a float between [0.0, 1.0]!')
self._mask_threshold = value
@@ -224,3 +229,53 @@ class MaskSeparationBase(separation_base.SeparationBase):
NotImplementedError: Cannot call base class!
"""
raise NotImplementedError('Cannot call base class!')
+
+ @classmethod
+ def from_json(cls, json_string):
+ """
+ Creates a new :class:`SeparationBase` object from the parameters stored in this JSON string.
+
+ Args:
+ json_string (str): A JSON string containing all the data to create a new :class:`SeparationBase`
+ object.
+
+ Returns:
+ (:class:`SeparationBase`) A new :class:`SeparationBase` object from the JSON string.
+
+ See Also:
+ :func:`to_json` to make a JSON string to freeze this object.
+
+ """
+ mask_sep_decoder = MaskSeparationBaseDecoder(cls)
+ return mask_sep_decoder.decode(json_string)
+
+
+class MaskSeparationBaseDecoder(separation_base.SeparationBaseDecoder):
+ """ Object to decode a :class:`MaskSeparationBase`-derived object from JSON serialization.
+ You should never have to instantiate this object by hand.
+ """
+
+ def __init__(self, separation_class):
+ self.separation_class = separation_class
+ json.JSONDecoder.__init__(self, object_hook=self._json_separation_decoder)
+
+ def _json_separation_decoder(self, json_dict):
+ if '__class__' in json_dict and '__module__' in json_dict:
+ json_dict, separator = self._inspect_json_and_create_new_instance(json_dict)
+
+ # fill out the rest of the fields
+ for k, v in json_dict.items():
+ if isinstance(v, dict) and nussl.constants.NUMPY_JSON_KEY in v:
+ separator.__dict__[k] = nussl.utils.json_numpy_obj_hook(v[nussl.constants.NUMPY_JSON_KEY])
+ elif isinstance(v, (str, bytes)) and nussl.audio_signal.__name__ in v: # TODO: test this
+ separator.__dict__[k] = nussl.audio_signal.AudioSignal.from_json(v)
+ elif k == 'result_masks':
+ # for mask_json in v:
+
+ separator.result_masks = [masks.MaskBase.from_json(itm) for itm in v]
+ else:
+ separator.__dict__[k] = v if not isinstance(v, unicode) else v.encode('ascii')
+
+ return separator
+ else:
+ return json_dict

diff:
@@ -25,6 +25,10 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
References:
Mel Frequency Cepstral Coefficients(MFCC): https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+ Volker Gnann, Martin Spiertz. "Source-filter based clustering for monaural blind source separation"
+ October 2002: https://www.researchgate.net/profile/Martin_Spiertz/publication/228567105_Source-filter_based
+ _clustering_for_monaural_blind_source_separation/links/0c960526b56dae0c69000000.pdf
+ scikit-learn's KMeans: http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
Parameters:
input_audio_signal (np.array): a 2-row Numpy matrix containing samples of the two-channel mixture.
@@ -33,20 +37,20 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
distance_measure (str): The type of distance measure to use in NMF - euclidean or divergence.
num_iterations (int): The number of iterations to go through in NMF.
random_seed (int): The seed to use in the numpy random generator
- convert_to_mono (bool): Given a stereo signal, convert to mono
+ kmeans_kwargs (dict): The kwargs for KMeans parameters.
+ convert_to_mono (bool): Given a stereo signal, convert to mono.
+ mask_type (object): A soft or binary mask object used for the source separation.
+ mfcc_range (int/list/tuple): The range of mfcc to use for clustering.
+ n_mfcc (int): The max number of mfccs to use.
Attributes:
- input_audio_signal (object): An Audio Signal object of the input audio signal
- clusterer (object): A KMeans object for clustering the templates and activations
- signal_stft (np.matrix): The stft data for the input audio signal
- templates_matrix (np.matrix): A Numpy matrix containing the templates matrix from running NMF on the
- current channel from the input signal stft data.
- activation_matrix (np.matrix): A Numpy matrix containing the activation matrix from running NMF on the
- current channel from the input signal stft data.
+ input_audio_signal (object): An Audio Signal object of the input audio signal.
+ clusterer (object): A KMeans object for clustering the templates and activations.
+ signal_stft (np.matrix): The stft data for the input audio signal.
labeled_templates (np.array): A Numpy array containing the labeled templates columns
- from the templates matrix for a particular source
- sources (np.array): A Numpy array containing the list of Audio Signal objects for each source
- masks (np.array): A Numpy array containing the lists of Binary Mask objects for each channel
+ from the templates matrix for a particular source.
+ sources (np.array): A Numpy array containing the list of Audio Signal objects for each source.
+ masks (np.array): A Numpy array containing the lists of Binary Mask objects for each channel.
"""
def __init__(self, input_audio_signal, num_sources, num_templates=50, distance_measure='euclidean',
@@ -66,8 +70,6 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.signal_stft = None
self.input_audio_signal = input_audio_signal
- self.templates_matrix = None
- self.activation_matrix = None
self.labeled_templates = None
self.sources = []
self.masks = []
@@ -84,21 +86,18 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
else:
raise ValueError('mfcc_range is not set correctly! Must be a tuple or list with min and max, or int (max)')
- # If provided, add random_seed to kmeans_kwargs
- if self.random_seed is not None:
- if self.kmeans_kwargs is None:
- self.kmeans_kwargs = {'random_state': self.random_seed}
- if 'random_state' not in self.kmeans_kwargs or self.kmeans_kwargs['random_state'] is None:
- self.kmeans_kwargs['random_state'] = self.random_seed
+ # If kmeans_kwargs does not include the 'random_state', use the random_seed instead. Else, use the value
+ # provided in the dictionary. If kmeans_kwargs is None, use the random_seed.
- # Set number of clusters to number of sources
- if self.kmeans_kwargs is None:
- self.kmeans_kwargs = {'n_clusters': self.num_sources}
- else:
- self.kmeans_kwargs['n_clusters'] = self.num_sources
+ self.kmeans_random_seed = kmeans_kwargs.pop('random_state', random_seed) \
+ if isinstance(kmeans_kwargs, dict) else random_seed
# Initialize the K Means clusterer
- self.clusterer = sklearn.cluster.KMeans(**self.kmeans_kwargs)
+ if isinstance(self.kmeans_kwargs, dict):
+ self.clusterer = sklearn.cluster.KMeans(n_clusters=self.num_sources, random_state=self.kmeans_random_seed,
+ **kmeans_kwargs)
+ else:
+ self.clusterer = sklearn.cluster.KMeans(n_clusters=self.num_sources, random_state=self.kmeans_random_seed)
def run(self):
""" This function calls TransformerNMF on the magnitude spectrogram of each channel in the input audio signal.
@@ -108,6 +107,25 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Returns:
self.masks (np.array): A list of binary mask objects that can be used to extract the sources
+
+ Example:
+
+ .. code-block:: python
+ :linenos:
+
+ signal = nussl.AudioSignal(path_to_input_file='input_name.wav')
+
+ # Set up and run Repet
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2) # Returns a binary mask by default
+ masks = nmf_mfcc.run()
+
+ # Get audio signals
+ sources = nmf_mfcc.make_audio_signals()
+
+ # output the background
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
"""
self.masks = []
self.input_audio_signal.stft_params = self.stft_params

diff:
@@ -11,6 +11,7 @@ import warnings
import nussl.audio_signal
import nussl.spectral_utils
import nussl.constants
+import nussl.separation.masks
class SeparationBase(object):
@@ -39,7 +40,7 @@ class SeparationBase(object):
# initialize to empty arrays so that we don't crash randomly
self.audio_signal.audio_data = np.array([])
- self.audio_signal.stft_data = np.array([])
+ self.audio_signal.stft_data = np.array([[]])
@property
def sample_rate(self):
@@ -110,14 +111,22 @@ class SeparationBase(object):
@staticmethod
def _to_json_helper(o):
if not isinstance(o, SeparationBase):
- raise TypeError
+ raise TypeError('SeparationBase._to_json_helper() got foreign object!')
d = copy.copy(o.__dict__)
for k, v in d.items():
if isinstance(v, np.ndarray):
d[k] = nussl.utils.json_ready_numpy_array(v)
- if isinstance(v, nussl.audio_signal.AudioSignal) or isinstance(v, nussl.spectral_utils.StftParams):
+ elif hasattr(v, 'to_json'):
d[k] = v.to_json()
+ elif isinstance(v, (list, tuple, set)) and any(hasattr(itm, 'to_json') for itm in v):
+ s = []
+ for itm in v:
+ if hasattr(itm, 'to_json'):
+ s.append(itm.to_json())
+ else:
+ s.append(itm)
+ d[k] = s
d['__class__'] = o.__class__.__name__
d['__module__'] = o.__module__
@@ -173,31 +182,23 @@ class SeparationBaseDecoder(json.JSONDecoder):
def __init__(self, separation_class):
self.separation_class = separation_class
- json.JSONDecoder.__init__(self, object_hook=self.json_separation_decoder)
-
- def json_separation_decoder(self, json_dict):
- """
- Helper method for :class:`SeparationBaseDecoder`. Don't you worry your pretty little head about this.
+ json.JSONDecoder.__init__(self, object_hook=self._json_separation_decoder)
- NEVER CALL THIS DIRECTLY!!
-
- Args:
- json_dict (dict): JSON dictionary provided by ``object_hook``
-
- Returns:
- A new :class:`SeparationBase`-derived object from JSON serialization
-
- """
- if '__class__' in json_dict:
+ def _inspect_json_and_create_new_instance(self, json_dict):
class_name = json_dict.pop('__class__')
module_name = json_dict.pop('__module__')
if class_name != self.separation_class.__name__ or module_name != self.separation_class.__module__:
- raise TypeError
+ raise TypeError('Expected {}.{} but got {}.{} from json!'.format(self.separation_class.__module__,
+ self.separation_class.__name__,
+ module_name, class_name))
# load the module and import the class
module = __import__(module_name)
class_ = getattr(module, class_name)
+ if '_audio_signal' not in json_dict:
+ raise TypeError('JSON string from {} does not have an AudioSignal object!'.format(class_name))
+
# we know 'input_audio_signal' is always the first argument
signal_json = json_dict.pop('_audio_signal') # this is the AudioSignal object
signal = nussl.audio_signal.AudioSignal.from_json(signal_json)
@@ -210,17 +211,35 @@ class SeparationBaseDecoder(json.JSONDecoder):
args = dict((k.encode('ascii'), json_dict[k]) for k in required_args)
# make a new instance of separation class
- seperator = class_(signal, **args)
+ separator = class_(signal, **args)
+
+ return json_dict, separator
+
+ def _json_separation_decoder(self, json_dict):
+ """
+ Helper method for :class:`SeparationBaseDecoder`. Don't you worry your pretty little head about this.
+
+ NEVER CALL THIS DIRECTLY!!
+
+ Args:
+ json_dict (dict): JSON dictionary provided by `object_hook`
+
+ Returns:
+ A new :class:`SeparationBase`-derived object from JSON serialization
+
+ """
+ if '__class__' in json_dict and '__module__' in json_dict:
+ json_dict, separator = self._inspect_json_and_create_new_instance(json_dict)
# fill out the rest of the fields
for k, v in json_dict.items():
if isinstance(v, dict) and nussl.constants.NUMPY_JSON_KEY in v:
- seperator.__dict__[k] = nussl.utils.json_numpy_obj_hook(v[nussl.constants.NUMPY_JSON_KEY])
+ separator.__dict__[k] = nussl.utils.json_numpy_obj_hook(v[nussl.constants.NUMPY_JSON_KEY])
elif isinstance(v, (str, bytes)) and nussl.audio_signal.__name__ in v: # TODO: test this
- seperator.__dict__[k] = nussl.audio_signal.AudioSignal.from_json(v)
+ separator.__dict__[k] = nussl.audio_signal.AudioSignal.from_json(v)
else:
- seperator.__dict__[k] = v if not isinstance(v, unicode) else v.encode('ascii')
+ separator.__dict__[k] = v if not isinstance(v, unicode) else v.encode('ascii')
- return seperator
+ return separator
else:
return json_dict

diff:
@@ -9,13 +9,13 @@ import os
def main():
path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_benchmark_file)
+
+ # Set random seed in NMF and KMeans to 0
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150, random_seed=0)
- output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
+ num_iterations=100, random_seed=0)
+ output_folder = os.path.abspath('nmf_mfcc_reference/new')
nmf_mfcc.run()
- np.save(os.path.join(output_folder, "benchmark_activation_matrix"), nmf_mfcc.activation_matrix)
- np.save(os.path.join(output_folder, "benchmark_templates_matrix"), nmf_mfcc.templates_matrix)
np.save(os.path.join(output_folder, "benchmark_labeled_templates"), nmf_mfcc.labeled_templates)
np.save(os.path.join(output_folder, "benchmark_masks"), nmf_mfcc.masks)

diff:
-5ee5d25fb0e0d4e73bd13e7a76e10ea77cc60ce9
\ No newline at end of file
+c74944e0486dee30085a755b662c03f19821ea95
\ No newline at end of file

diff:
-e80ca319a52501f5b340d3d34821120f19a790d5
\ No newline at end of file
+198eb8502ac846b3f3f28e3b9986edce15df270d
\ No newline at end of file

change_type: DELETE

diff:
-650d69d03451df290fda2fff1ba0b59e23b23a89
\ No newline at end of file

diff:
@@ -57,3 +57,16 @@ class TestJson(unittest.TestCase):
f = nussl.RepetSim.from_json(j)
worked = r == f
return worked
+
+ def test_nmf_mfcc(self):
+ path = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
+ a = nussl.AudioSignal(path)
+ n = nussl.NMF_MFCC(a, num_sources=2)
+ n()
+
+ j = n.to_json()
+ f = nussl.NMF_MFCC.from_json(j)
+ worked = n == f
+ return worked
+
+

diff:
@@ -150,5 +150,3 @@ class NMFMFCCUnitTests(unittest.TestCase):
assert np.all(benchmark_labeled_templates == nmf_mfcc.labeled_templates)
assert np.all(benchmark_masks == nmf_mfcc.masks)
assert np.all(benchmark_sources == nmf_mfcc.sources)
-
-


message: Made changes to documentation, seed, and tests.

diff:
@@ -88,7 +88,6 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
# If kmeans_kwargs does not include the 'random_state', use the random_seed instead. Else, use the value
# provided in the dictionary. If kmeans_kwargs is None, use the random_seed.
-
self.kmeans_random_seed = kmeans_kwargs.pop('random_state', random_seed) \
if isinstance(kmeans_kwargs, dict) else random_seed